diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..8f6f237112f5389501dec3520a03fd5b8efdcc61 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# SynthScope secrets template — copy to `.env` (gitignored) and fill in. +# The backend auto-loads `.env` at startup (interface/backend/config.py:_load_dotenv). +# Real shell env vars / secrets-manager values OVERRIDE this file. + +# SOTA image-generator API keys (for the "SOTA vs Detectors" demo). Each is optional; +# a generator is greyed-out in the UI if its key is unset. +GEMINI_API_KEY= # Nano Banana (Gemini 2.5 Flash Image) -> aistudio.google.com/apikey +OPENAI_API_KEY= # GPT Image (gpt-image-1) -> platform.openai.com/api-keys +XAI_API_KEY= # Grok (grok-2-image) -> console.x.ai +FAL_KEY= # FLUX img2img via fal.ai -> fal.ai/dashboard/keys + +# Ideogram 4.0 runs LOCALLY (open-weight) — no paid API key. HF_TOKEN is only needed ONCE to +# download the license-gated weights (accept the license at hf.co/ideogram-ai/ideogram-4-nf4-diffusers). +HF_TOKEN= # free Hugging Face token, for gated open-weight downloads + +# Optional path overrides (defaults assume the cn023 node layout) +# SYNTHSCOPE_CKPT_DIR=/hddstore/ssalehi/SynthScope/checkpoints diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..74dd4da04f4762477adaf7a3c8988ac13b0bfef4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +docs/images/datasets.jpg filter=lfs diff=lfs merge=lfs -text +docs/images/detect.jpg filter=lfs diff=lfs merge=lfs -text +docs/images/genbench.jpg filter=lfs diff=lfs merge=lfs -text +docs/images/generator.jpg filter=lfs diff=lfs merge=lfs -text +docs/images/leaderboard.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000000000000000000000000000000000000..6e03d2f3b3c91cf472bcb88e7d4417d1941b0a9e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,39 @@ +name: Bug report +description: Something doesn't work as expected +labels: ["bug"] +body: + - type: textarea + id: what-happened + attributes: + label: What happened? + description: A clear description of the bug, including the full error/traceback. + placeholder: Paste the traceback here. + validations: + required: true + - type: textarea + id: repro + attributes: + label: How to reproduce + description: The exact command or minimal code snippet (model, dataset, overrides). + placeholder: | + bash experiments/aide.sh + # or + import synthscope; synthscope.load("fat_former").predict("x.png") + render: shell + validations: + required: true + - type: input + id: model + attributes: + label: Detector / dataset + description: Which `model_type` and/or dataset config is involved? + - type: textarea + id: env + attributes: + label: Environment + description: OS, Python, PyTorch + CUDA versions, GPU, and `synthscope` / commit hash. + placeholder: | + python -c "import torch, synthscope; print(synthscope.__version__, torch.__version__, torch.version.cuda)" + render: shell + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..d8e6aeeb6a48e6c9a3fb73ffd4465cfd51a48979 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Questions & discussion + url: https://github.com/SMSD75/SynthScope/discussions + about: Ask usage questions, share results, or propose ideas before opening an issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000000000000000000000000000000000000..8d440d4d7533609002d7dbb672c8abbb344c97fc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,27 @@ +name: Feature request / new detector or dataset +description: Propose a new detector, dataset, or capability +labels: ["enhancement"] +body: + - type: dropdown + id: kind + attributes: + label: Type + options: + - New detector (method) + - New dataset / benchmark + - API / usability + - Other + validations: + required: true + - type: textarea + id: proposal + attributes: + label: What would you like? + description: Describe the method/dataset/feature. For a new detector, link the paper and official repo. + validations: + required: true + - type: textarea + id: value + attributes: + label: Why is it useful? + description: What does it add to the benchmark (e.g. a generator family or modality not yet covered)? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..5eba98c144cf7f62fecd49b35e0a06a6118dd2e9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ + + +## Summary + + +## Type of change +- [ ] New detector (`model_type`) +- [ ] New dataset / benchmark +- [ ] Bug fix +- [ ] Docs +- [ ] Other + +## Checklist +- [ ] Ran the affected experiment/eval and pasted the resulting `Test Metrics` below. +- [ ] For a new detector: added a `models///` folder with a method `README.md`, + `conf/model/...` + `conf/trainer/...` groups, an `experiments/*.sh` recipe, and (if it can + run from the API) a registry entry in `interface/backend/registry.py`. +- [ ] `ruff check .` passes and `python -m compileall` is clean. +- [ ] Updated the reproduction table / docs if results changed. + +## Results + +``` +``` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..82a347bb72d55e953b2eca1befa52a8f9562f16a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +# Lightweight, always-green-on-day-one checks. We deliberately do NOT build the heavy model +# stack (torch + large backbones + dataset downloads) on GitHub runners; functional smoke tests +# run on a GPU machine. See tests/ for the local smoke suite. +jobs: + compile: + name: Byte-compile (syntax) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Compile all Python sources + run: python -m compileall -q synthscope synthscope interface models benchmark *.py + + leaderboard: + name: Benchmark submissions + leaderboard + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + # Pure-stdlib check: validates every submissions/*.json and that LEADERBOARD.md is current. + - run: python benchmark/leaderboard.py --check + + lint: + name: Ruff (errors only) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: pip install "ruff>=0.4" + # Lint the maintained library surface (config + ignores in pyproject.toml [tool.ruff]). + # Repo-wide syntax is covered by the byte-compile job above. + - run: ruff check synthscope synthscope + + package: + name: Build + metadata check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: pip install "build>=1" "twine>=5" + - run: python -m build + - run: twine check dist/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2a9e4107bd1c201089e6fba3d3c9a48d725dba65 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +*.png +*.pth +.cursor +__pycache__ +wandb/ +*.log +.codex +.vscode +outputs/ +.cache/ +.claude/ +# packaging / tooling artifacts +build/ +dist/ +*.egg-info/ +.ruff_cache/ +.pytest_cache/ + +# Secrets — never commit; use .env.example as the template +.env +.env.local +.env.*.local + +# Large shared HF/torch cache (node-local downloads) +.cache_shared/ + +# arXiv trending candidates (regenerated daily by fetch_arxiv_trending.py) +repro/trending_arxiv_candidates.json + +# generator benchmark runs: keep manifest/results/RESULTS.md, not the (large) image sets +benchmark/generator_runs/*/real/ +benchmark/generator_runs/*/fake/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..fe1c0e3df699777aaa7d27c0de2c6978b81218cf --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +# Codex Project Rules + +- For this SynthScope project, always run Python/project commands inside the `SynthScope` conda environment, for example: + `/home/ssalehi/miniconda3/bin/conda run -n SynthScope `. +- For this SynthScope project, discuss the intended approach and changes first. Only apply file/code changes after the user explicitly asks to run or make the changes. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000000000000000000000000000000000..cff09dbff612385ea8f237847a7fd99961f301d7 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,24 @@ +cff-version: 1.2.0 +message: "If you use SynthScope in your research, please cite it." +title: "SynthScope: A Unified Library and Benchmark for AI-Generated Image and Video Detection" +abstract: >- + SynthScope re-implements a range of state-of-the-art AI-generated (synthetic) image and video + detectors behind a single configuration system, training loop, datasets, transforms, and + metrics, so that different methods can be trained and benchmarked under identical conditions, + with results reported per generator/source. +type: software +authors: + # TODO(maintainer): verify the author name(s) and affiliation, and add ORCID(s). + - family-names: "Salehi Dehnavi" + given-names: "Seyed" + email: "s.salehidehnavi@uva.nl" + affiliation: "University of Amsterdam" +repository-code: "https://github.com/SMSD75/SynthScope" +url: "https://github.com/SMSD75/SynthScope" +license: Apache-2.0 +version: "0.1.0" +keywords: + - AI-generated image detection + - synthetic media detection + - deepfake detection + - benchmark diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..95e6ab120fe1ebf34f7475e2ce28ea1beec567e1 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,33 @@ +# Contributor Code of Conduct + +## Our pledge + +We as members, contributors, and maintainers pledge to make participation in the SynthScope community a +harassment-free experience for everyone, regardless of age, body size, visible or invisible +disability, ethnicity, sex characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, religion, or sexual +identity and orientation. + +## Our standards + +Examples of behavior that contributes to a positive environment: + +- Being respectful of differing opinions, viewpoints, and experiences. +- Giving and gracefully accepting constructive feedback. +- Focusing on what is best for the community and the science. + +Unacceptable behavior includes harassment, insulting or derogatory comments, personal or political +attacks, publishing others' private information, and other conduct that could reasonably be +considered inappropriate in a professional setting. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project +maintainers at **s.salehidehnavi@uva.nl**. All complaints will be reviewed and investigated +promptly and fairly. Maintainers are responsible for clarifying and enforcing these standards and +may take any action they deem appropriate, up to and including a temporary or permanent ban. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), +version 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..4fa482520349ccf4122e84c33259d2c10f6511aa --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,71 @@ +# Contributing to SynthScope + +Thanks for your interest! SynthScope aims to be the **unified, reproducible benchmark** for +AI-generated image & video detection. The most valuable contributions are **new detectors**, +**new datasets/generators**, and **independently reproduced results**. + +## Getting set up + +```bash +git clone https://github.com/SMSD75/SynthScope.git +cd SynthScope +pip install -e ".[dev]" # editable install + contributor tooling +# PyTorch: install the build matching your CUDA from https://pytorch.org if needed. +# Some detectors also need OpenAI CLIP: pip install git+https://github.com/openai/CLIP.git +``` + +Before opening a PR: + +```bash +ruff check synthscope # lint the library surface (errors only; see pyproject [tool.ruff]) +python -m compileall synthscope interface models *.py +pytest tests/ -k smoke # where data/weights are available +``` + +## Adding a new detector + +The framework is model-agnostic: the trainer only needs a `compute_loss(batch)` method, where a +batch is `(image, label, folder_id)` and `folder_id` is the generator/source used for +per-generator metrics. To add a method `foo`: + +1. **Model** — create `models//foo/foo.py` with an `nn.Module` exposing + `compute_loss(batch) -> {"loss": ..., "outputs": ...}`. Add a short method `README.md`. +2. **Dispatch** — register `model_type=foo` in `SynthScopeModelTrainer._build_model` (`trainer.py`). +3. **Config** — add `conf/model//foo.yaml` and `conf/trainer//foo.yaml`. +4. **Recipe** — add `experiments//foo.sh` with the env vars + launch command. +5. **(Optional) API/demo** — add a `ModelSpec` to `interface/backend/registry.py` so the detector + is runnable via `synthscope.load("foo")`, the CLI, and the web demo (preprocessing, checkpoint + + any official-weights remap, and the forward → P(fake) convention). +6. **Results** — run it and paste the logged `Test Metrics` in the PR. New numbers should trace to + a real run, never be estimated (see the reproduction table convention in the README). + +## Adding a dataset + +Add a `dataset` class + data module in `dataset.py` (auto-download → common `real`/`fake` layout) +and a `conf/dataset//.yaml`. Keep paths overridable via config/env — no hard-coded +absolute paths. + +## Submitting a benchmark result + +Add one JSON file under [`benchmark/submissions/`](benchmark/submissions/) (see +[`benchmark/schema.json`](benchmark/schema.json) and the protocol in +[`benchmark/README.md`](benchmark/README.md)). Prefer `verified` results produced by +`benchmark/evaluate.py`; for `external` numbers, set `source`. Regenerate and commit the table: + +```bash +python benchmark/evaluate.py --model --track --data --out benchmark/submissions/.json +python benchmark/leaderboard.py # updates LEADERBOARD.md +python benchmark/leaderboard.py --check # what CI runs +``` + +## Style + +- Match the surrounding code; keep imports and helper/class definitions at module top level. +- Keep changes focused; one detector/dataset/fix per PR. +- Be honest about faithfulness: if a re-implementation diverges from the paper, document it (as the + reproduction notes already do). + +## Reporting bugs / proposing features + +Use the issue templates. For questions, open a Discussion. By contributing you agree your work is +licensed under the project's [Apache-2.0 license](LICENSE). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..51451c10b5f8b8d2d7d2144cb3a9c92b19e6bf01 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# SynthScope web page as a Hugging Face Docker Space (CPU hardware, e.g. "CPU Upgrade": 8 vCPU, 32 GB). +# Build context = the repository root (push_space.py copies this file to the root of the Space repo). +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends git ffmpeg libgl1 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* +# Spaces run the container as user 1000 +RUN useradd -m -u 1000 user +USER user +ENV HOME=/home/user PATH=/home/user/.local/bin:$PATH PYTHONUNBUFFERED=1 \ + SYNTHSCOPE_CKPT_DIR=/home/user/ckpts SYNTHSCOPE_CACHE_DIR=/home/user/cache HF_HOME=/home/user/cache/hf \ + SYNTHSCOPE_MAX_UPLOAD_MB=20 SYNTHSCOPE_MAX_RESIDENT_MODELS=2 XFORMERS_DISABLED=1 +WORKDIR /home/user/app + +# CPU-only PyTorch first (much smaller than the CUDA build), then SynthScope with the web extras +RUN pip install --no-cache-dir --user torch torchvision --index-url https://download.pytorch.org/whl/cpu +COPY --chown=user . . +RUN pip install --no-cache-dir --user -e ".[web]" "git+https://github.com/openai/CLIP.git" + +EXPOSE 7860 +CMD ["bash", "deploy/hf-space/start.sh"] diff --git a/LEADERBOARD.md b/LEADERBOARD.md new file mode 100644 index 0000000000000000000000000000000000000000..26c9da472540b2c9928c261a408633521b221376 --- /dev/null +++ b/LEADERBOARD.md @@ -0,0 +1,36 @@ +# SynthScope Leaderboard (v1) + +_Auto-generated by `benchmark/leaderboard.py` — do not edit by hand._ See [benchmark/README.md](benchmark/README.md) for the protocol and how to submit. + +This is the **standardized, versioned** table: a few tracks with a fixed protocol that anyone can submit to. The web page's **Leaderboard** tab is broader (every paper benchmark, 30+ groups) and the repo supports many more datasets, which download and organize themselves: see [docs/BENCHMARKS.md](docs/BENCHMARKS.md). + +Metrics are percentages. **mAP** = mean AP over generators. Each track is ranked by its primary metric (in bold). + +## ForenSynths — image (ProGAN-trained, cross-generator) + +| Rank | Method | Eval | Acc | **mAP** | AP | AUROC | Verified | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 1 | FatFormer | official-weights | 98.39 | 99.70 | — | — | ✓ | ForenSynths-8 subset; official released weights | +| 2 | CO-SPY (official weights) | official-weights | 95.12 | 98.71 | — | — | ✓ | ForenSynths-8; vendored official architecture | +| 3 | UnivFD / CLIP | frozen-probe | — | 97.20 | — | 97.00 | ✓ | ForenSynths; frozen CLIP ViT-L/14 probe (ProGAN) | +| 4 | CO-SPY (from scratch) | from-scratch | — | 96.98 | — | 96.79 | ✓ | ForenSynths; SynthScope re-implementation trained on ProGAN | +| 5 | CNNDetection | from-scratch | — | 96.80 | — | 96.60 | ✓ | ForenSynths; ResNet-50 fine-tune (ProGAN) | +| 6 | DINOv2 probe | frozen-probe | — | 95.03 | — | 94.86 | ✓ | ForenSynths; frozen DINOv2 ViT-L/14 probe | +| 7 | SAFE | from-scratch | — | 94.13 | — | 93.68 | ✓ | ForenSynths; trained from scratch | +| 8 | AIDE | official-weights | — | 92.08 | — | — | ✓ | ForenSynths-8; mAP reproduces, acc@0.5 miscalibrated (single-view vs official 5-view DCT) | +| 9 | NPR | official-weights | 91.83 | 87.09 | — | — | ✓ | ForenSynths-8; official ProGAN weights | + +## GenImage SD-v1.4 — image + +| Rank | Method | Eval | Acc | mAP | AP | **AUROC** | Verified | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 1 | VIB-Net | from-scratch | — | 99.33 | — | 99.33 | ✓ | GenImage SD-v1.4 single real/fake pool (mAP == AUROC by construction) | +| 2 | SPAI (official weights) | official-weights | — | — | 94.01 | 98.45 | ✓ | GenImage SD-v1.4; vendored official architecture, fixed-res 2x2x224 | +| 3 | SPAI (from scratch) | from-scratch | — | — | — | 97.66 | ✓ | DMID Latent-Diffusion vs COCO; SynthScope fixed-resolution variant | + +## GenVideo-Val — video + +| Rank | Method | Eval | Acc | mAP | **AP** | AUROC | Verified | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 1 | ReStraV | frozen-probe | — | — | 96.67 | 97.06 | ✓ | Frozen DINOv2 trajectory geometry + MLP | +| 2 | D3 | training-free | — | — | 95.60 | 93.60 | ✓ | Training-free frozen CLIP 2nd-order trajectory | diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9844746f3ff48b7ddc7cf158a457fe0f1ea93b23 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, excluding + those notices that do not pertain to any part of the Derivative + Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Seyed Salehi Dehnavi and the SynthScope contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES.md b/LICENSES.md new file mode 100644 index 0000000000000000000000000000000000000000..80e2b304968057f6f5d35a1a1f1d504637133c29 --- /dev/null +++ b/LICENSES.md @@ -0,0 +1,65 @@ +# Licenses and attribution + +The **SynthScope framework** — the training/eval pipeline, the unified config system, the dataset +modules, the metrics/transform/logging stack, the `synthscope` Python API, and the web interface — is +released under the **Apache License 2.0** (see [LICENSE](LICENSE)). + +SynthScope **re-implements** a range of published detectors and, for a few methods, **vendors** the +authors' original network code so their released weights can be loaded faithfully. Those +third-party components, and every **pretrained checkpoint / backbone** SynthScope loads, remain under +**their original licenses**. SynthScope does not relicense them. + +> ⚠️ **Before redistributing any model weights** (e.g. via a public model zoo / Hugging Face Hub), +> confirm that the upstream license permits redistribution and your intended use. Several +> AI-generated-image-detection paper repos and some pretrained backbones are **research- or +> non-commercial-only**, and some weights are **gated** (require accepting an upstream license). +> The table below is a starting point — verify each entry against its source. (`⚠ verify` = please +> confirm and fill in from the official repo before relying on it.) + +## Detectors + +| `model_type` | Method (venue) | Upstream code | Code license | Released weights SynthScope can load | Weights license | +| --- | --- | --- | --- | --- | --- | +| `cnn` | CNNDetection (CVPR'20) | PeterWang512/CNNDetection | ⚠ verify | ProGAN-trained (SynthScope-trained here) | ⚠ verify | +| `clip` | UnivFD (CVPR'23) | WisconsinAIVision/UniversalFakeDetect | ⚠ verify | — (SynthScope-trained probe) | n/a | +| `npr` | NPR (CVPR'24) | chuangchuangtan/NPR-DeepfakeDetection | ⚠ verify | **official** ProGAN weights | ⚠ verify | +| `fat_former` | FatFormer (CVPR'24) | Michel-liu/FatFormer (verify) | ⚠ verify | **official** weights | ⚠ verify | +| `aide` | AIDE (ICLR'25) | ⚠ verify (official repo) | ⚠ verify | **official** weights | ⚠ verify | +| `co_spy` | CO-SPY (CVPR'25) | ⚠ verify (official repo) | ⚠ verify | **official** weights | ⚠ verify | +| `spai` | SPAI (CVPR'25) | mever-team/spai (verify) | ⚠ verify | **official** weights | ⚠ verify | +| `vib_net` | VIB-Net (CVPR'25) | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a | +| `safe` | SAFE (KDD'25) | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a | +| `pgc` | PGC | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a | +| `cure` | CuRe (arXiv 2609.14316) | manic-cui/CuRe | Apache-2.0 (repo `LICENSE`) | **official** 25 MB adapter (`cure_adapter.pt`, loads as `cure.pth`) | Apache-2.0 (shipped in the repo; verify before redistributing) | +| `dinov2` | DINOv2 probe | facebookresearch/dinov2 | Apache-2.0 (verify) | DINOv2 backbone | Apache-2.0 (verify) | +| `dinov3` | DINOv3 probe | facebookresearch/dinov3 | DINOv3 License (gated) | DINOv3 backbone | **gated** — DINOv3 License | +| `adapted_dino` | Adapter-tuned DINOv2 (SynthScope) | this repo | Apache-2.0 | — (SynthScope-trained) | n/a | +| `d3` | D3 (ICCV'25) | ⚠ verify (official repo) | ⚠ verify | training-free (no weights) | n/a | +| `restrav` | ReStraV | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a | +| `stall` | STALL (CVPR'26) | ⚠ verify (official repo) | ⚠ verify | training-free; needs DINOv3 backbone | gated (DINOv3) | + +## Backbones / foundation models (downloaded on first use) + +| Backbone | Used by | License | +| --- | --- | --- | +| OpenAI CLIP (ViT-L/14) | `clip`, `fat_former`, `d3` | MIT (verify) | +| OpenCLIP (ConvNeXt-XXL, LAION) | `aide` | code MIT; weights per LAION / model card (verify) | +| DINOv2 (ViT-L/14, ViT-S/14) | `dinov2`, `adapted_dino`, `restrav` | Apache-2.0 (verify) | +| DINOv3 (ViT-L/16) | `dinov3`, `stall` | **gated** — DINOv3 License (non-commercial; accept upstream) | +| SigLIP | `co_spy` | Apache-2.0 (verify) | +| MFM ViT-B/16 | `spai` | ⚠ verify | +| Stable Diffusion VAE | `co_spy` | CreativeML Open RAIL-M (verify) | +| Perception Encoder Core L/14-336 (`facebook/PE-Core-L14-336`) | `cure` | Apache-2.0 (Meta); `official_arch.py` vendors the CuRe/Meta backbone code — see its header | + +## Datasets + +The dataset auto-downloaders fetch third-party datasets (GenImage, ForenSynths/CNNDetection, +AIGIBench, Co-Spy-Bench, DiTFake, UniversalFakeDetect, GenVideo, …). Each dataset is governed by +its **own** terms / license at the source; SynthScope only automates retrieval and restructuring. Review +and comply with each dataset's terms before use or redistribution. + +--- + +*This file is a good-faith attribution map, not legal advice. Maintainers: replace every `⚠ verify` +with the confirmed upstream repository URL and license before publishing a model zoo or any +redistributed weights.* diff --git a/README.md b/README.md index acfb111775d798fb97162d4849a9719740befe95..33bd2a221023e51204635669e86df388413ca196 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,22 @@ --- title: SynthScope -emoji: 📚 -colorFrom: blue -colorTo: red +emoji: 🔎 +colorFrom: indigo +colorTo: blue sdk: docker +app_port: 7860 pinned: false +license: apache-2.0 +short_description: Is this image AI-generated? Compare published detectors. --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# SynthScope + +Check whether an image or video is AI-generated with published detectors, see what each detector looks at, and compare how +they hold up against today's image generators. This Space runs on CPU: a detector takes a few seconds per image. + +Code, benchmarks and documentation: **https://github.com/SMSD75/SynthScope** + +Each detector and its weights keep the license chosen by their authors (see the repository's `LICENSES.md`); this demo is +free and non-commercial. If you are an author and would like your method removed or credited differently, please open an +issue on GitHub. diff --git a/TRENDING.md b/TRENDING.md new file mode 100644 index 0000000000000000000000000000000000000000..15716b18556901dae1011272b930ce7e6ae8d0ab --- /dev/null +++ b/TRENDING.md @@ -0,0 +1,84 @@ +# 📈 What's Trending — AI-generated content detection + +*High-quality recent papers for image & video AIGC detection. High-quality = (1) a solid public GitHub repository, and (2) strong results versus previous methods. Auto-refreshed daily. Last updated: 2026-10-10.* + +Entries marked _in SynthScope_ are integrated in this repo; the rest are tracked, with the reason an integration is pending noted in their _Results_ line. See [docs/ADD_DETECTOR.md](docs/ADD_DETECTOR.md) for how a paper is integrated and reproduced. + +## Image detectors +- **[CuRe: Learning Continuous Source Responses for Generalizable AI-Generated Image Detection](https://github.com/manic-cui/CuRe)** (manic-cui · arXiv · 2026) — [code](https://github.com/manic-cui/CuRe) · _in SynthScope: `cure`_ + Reframes backbone adaptation as regressing real-generated MIXING RATIOS instead of binary classification (finer supervision), with a compact source-response subspace that suppresses nuisance variation/shortcut cues. Frozen Perception-Encoder (PE-Core L/14-336) backbone + LoRA + subspace + linear head; the 25 MB adapter ships in the repo and infer.py scores a single image. + _Results: 89.7% average balanced accuracy over ten public benchmarks, +5.2 pts over the second-best method, robust to common degradations (paper). [Integrated in SynthScope as `cure` (merged to main 2026-10-03): SynthScope's class is identical to the official model (395/395 state-dict tensors, logit diff 4.8e-6, 0 verdict flips). SynthScope-protocol results with the official adapter: UFD-8 93.9 / ForenSynths-13 82.0 / GenImage-8 85.2 balanced acc (SD-v1.4, its training set, 99.7). AIGIBench REPRODUCED: on the paper's own 20 full-image-synthesis subsets (full test data) the port scores 92.85 balanced acc / 97.48 AUROC vs the paper's 92.8 / 97.5, every subset within 0.10 pt (per-subset mean abs diff 0.03; the paper excludes the 5 face-swap subsets, which is why all 25 gives 85.95). Still unconfirmed: UnivFD (aggregation unspecified), AIGCDetect (data only on cn023, not run), and 7 of the paper's 10 benchmarks (not in SynthScope).]_ +- **[PGC: Peak-Guided Calibration for AI-Generated Image Detection](https://github.com/xiaoyu6868/PGC)** (Zhou et al. · 2026) — [code](https://github.com/xiaoyu6868/PGC) · _in SynthScope: `pgc`_ + LoRA-tuned DINOv2 RGB stream + a YCbCr-quantization-residual CNN, fused by a peak-guided (log-sum-exp) calibration. Strong cross-generator transfer. + _Results: ForenSynths 96.25 mAP / GenImage 100 mAP (official weights)._ +- **[PROBE: Probing Generative Space for Generalizable AIGI Detection](https://github.com/Amamiya-C/PROBE-AIGI-Detection)** (arXiv · 2026) — [code](https://github.com/Amamiya-C/PROBE-AIGI-Detection) · _in SynthScope: `probe`_ + Boosts cross-generator generalization by augmenting training with images sampled from probed generative-space variations; the detector is a fine-tuned DINOv2 (sliding-patch) classifier. Released weights on ModelScope. + _Results: PROBE-DINOv2 93.9% avg balanced-acc over 7 benchmarks (paper). SynthScope reproduces the official weights: GenImage 98.54 / Synthbuster 99.27 / UFD 95.95 mAP (mAUROC 96-99). [Integration on branch trending/probe, under review.]_ +- **[DiCoME: Divide-and-Conquer Multi-View Evidential Learning for Deepfake Detection](https://github.com/kxl0825/DiCoME)** (arXiv · 2026) — [code](https://github.com/kxl0825/DiCoME) + Disentangles semantic vs structural-artifact views and fuses them with evidential learning to counter the 'semantic masking effect'. Released checkpoint (HF). + _Results: SOTA cross-dataset deepfake generalization (DeepfakeBench protocol). [face-deepfake domain; SynthScope lacks those datasets locally — integration pending data.]_ +- **[Images as Tables: In-Context AIGI Detection with TabPFN](https://github.com/jpwalter30/Towards-Generalizable-Detection-of-AI-Generated-Images)** (arXiv · 2026) — [code](https://github.com/jpwalter30/Towards-Generalizable-Detection-of-AI-Generated-Images) + Frozen DINOv3 CLS feature -> PCA(500) -> TabPFN in-context tabular classification; strong few-shot/low-data generalization with no task-specific training. + _Results: Generalizable detection in the low-data regime (few labeled examples per new generator). [integration pending: in-context TabPFN paradigm — a clean SynthScope forward->logit needs the released fitted TabPFN+PCA+labeled-support-set checkpoint baked in (GDrive), plus an external version-sensitive tabpfn dependency; a different paradigm from the trained-detector template, and the headline is a low-data/few-shot niche (multi-multi acc 0.8765 / AUROC 0.9501 @625 samples) rather than full-data SOTA. Verified repo 2026-06-20: weights on GDrive, frozen DINOv3->PCA(500)->TabPFN.]_ +- **[Co-Spy: Fusing Semantic and Artifact Cues for Synthetic Image Detection](https://github.com/Megum1/Co-Spy)** (Cheng et al. · CVPR · 2025) — [code](https://github.com/Megum1/Co-Spy) · _in SynthScope: `co_spy`_ + Adaptive fusion of a SigLIP semantic stream and an SD-VAE artifact-residual stream. + _Results: Co-Spy-Bench ~96 mAP; SynthScope retrain reaches 96.6 (beats released weights on that bench)._ +- **[SAFE: High-Frequency Artifact Detector with Targeted Augmentation](https://github.com/Ouxiang-Li/SAFE)** (Ouxiang Li et al. · KDD · 2025) — [code](https://github.com/Ouxiang-Li/SAFE) · _in SynthScope: `safe`_ + Lightweight DWT high-frequency artifact detector with a carefully designed augmentation recipe. + _Results: SynthScope-trained reproduction (served): ForenSynths-8 97.79 / UFD-8 98.44 / GenImage-8 99.51 / DiTFake 99.96 mAP — beats the official checkpoint on GenImage._ +- **[AIDE: ConvNeXt Semantic + Low-Level DCT/SRM Cues](https://arxiv.org/abs/2406.19435)** (Yan et al. · ICLR · 2025) — [arXiv](https://arxiv.org/abs/2406.19435) · _in SynthScope: `aide`_ + Fuses ConvNeXt semantic features with low-level DCT/SRM frequency cues; native-resolution preprocessing. + _Results: GenImage 97.12 mAP / AIGCDetect 97.94 mAP (official weights)._ +- **[SPAI: Spectral AI-Image Detection (any-resolution)](https://github.com/mever-team/spai)** (Karageorgiou et al. · CVPR · 2025) — [code](https://github.com/mever-team/spai) · _in SynthScope: `spai`_ + Spectral Reconstruction Similarity + Spectral Context Attention; detects at native/arbitrary resolution. + _Results: SynthScope serves the official weights at native resolution (the faithful any-res path): GenImage 94.22 / Synthbuster 96.75 mAP — reproduces & beats the vendored-official 91.33._ +- **[FatFormer: Forgery-Aware Adaptive Transformer](https://github.com/Michel-liu/FatFormer)** (Liu et al. · CVPR · 2024) — [code](https://github.com/Michel-liu/FatFormer) · _in SynthScope: `fat_former`_ + Frozen CLIP + forgery-aware adapters + learnable text-prompt context. + _Results: ForenSynths-8 99.7 mAP (official weights)._ +- **[NPR: Neighboring-Pixel Relationships](https://github.com/chuangchuangtan/NPR-DeepfakeDetection)** (Tan et al. · CVPR · 2024) — [code](https://github.com/chuangchuangtan/NPR-DeepfakeDetection) · _in SynthScope: `npr`_ + Small ResNet on neighboring-pixel residuals that expose generator up-sampling artifacts. + _Results: GANGen 96.49 mAP / DiffusionForensics 99.4 mAP (official weights)._ +- **[UnivFD / CLIP: Towards Universal Fake Image Detectors](https://github.com/WisconsinAIVision/UniversalFakeDetect)** (Ojha et al. · CVPR · 2023) — [code](https://github.com/WisconsinAIVision/UniversalFakeDetect) · _in SynthScope: `clip`_ + Linear probe on frozen CLIP ViT-L/14 features — a simple, strong cross-generator baseline. + _Results: UniversalFakeDetect 95.66 mAP; SynthScope-trained 96.44._ +- **[MoE-JEPA: Unifying Semantic Priors and High-Frequency Traces (V-JEPA for synthetic image forensics)](https://github.com/ALCOR-Lab-DIAG/MoE-JEPA)** (ALCOR-Lab-DIAG · ECCV'26 Workshop (AI for Multimedia Forensics) · 2026) — [code](https://github.com/ALCOR-Lab-DIAG/MoE-JEPA) + Frozen V-JEPA 2 ViT-L backbone with a Residual Mixture-of-Experts injected into the last blocks, a Bayar-conv noise branch and Gated-Attention MIL pooling; uses JEPA 'world understanding' as a semantic prior. + _Results: 95.54% acc / 94.21 F1 on SID-Set (300K images), outperforming much larger models (paper). [integration pending (assessed 2026-10-02): the authors' trained weights are NOT linked (only the frozen V-JEPA 2 backbone is downloaded separately) and there is no single-image inference entrypoint; can only be retrained. Re-checked 2026-10-05: unchanged.]_ +- **[RAID: Towards Robust AI-Generated Image Detection with Bit-Reversed Images](https://github.com/renxi-seu/RAID)** (renxi-seu · arXiv · 2026) — [code](https://github.com/renxi-seu/RAID) + Detects via bit-planes: a bit-reversed image, gradient-based patch selection and a small convolutional classifier, with a theoretical analysis; ~100x faster than counterparts. + _Results: Claims to outperform existing methods on 40+ benchmarks across cross-generator, cross-dataset and zero-shot settings (paper; README gives no numbers). [integration pending (assessed 2026-10-02): the SD-v1.4-trained weights are only shared via Baidu Cloud (not fetchable programmatically), and the README leaves the architecture and patch-score aggregation unspecified; revisit if a direct weight link appears. Re-checked 2026-10-05: unchanged.]_ +- **[Fleet: Few Shots Lead Effective AI-generated Image Detection](https://github.com/ICTMCG/Fleet)** (ICTMCG · arXiv · 2026) — [code](https://github.com/ICTMCG/Fleet) + Dynamic few-shot adaptation: 'constrained routing correction' redirects novel AI samples away from non-AI-dominated routes in decoupled subspaces (DINOv3 backbone), so the detector adapts to rapidly-evolving/commercial generators from a handful of labeled examples. Introduces the Treasure benchmark (64 generators, 360k images, 20 closed-source commercial engines). + _Results: On a modern generator (Doubao Seedream 4.0), 20.4% -> 73.1% with only 10-shot adaptation, where static SOTA collapses (paper). Repo now live with code (re-listed 2026-10-03; it had been pruned 2026-07-04 while an empty placeholder). [integration pending (assessed 2026-10-03): not a single-checkpoint detector — the Hugging Face repo wangjiaan/Fleet holds NINE per-target-generator few-shot adapted models (ADM, FLUX.2, GPT4O, HunyuanImage-3.0, NextStep, Qwen-Image, StarGAN, wan2.5, ...; 1.3 GB each), the generic stage-1 base is only on Baidu Pan, inference requires few-shot support data per target generator, and the Treasure benchmark is gated. No clean forward->logit that represents the method; revisit if a generic checkpoint appears on HF.]_ + +## Video detectors +- **[D3: Disentangled Detail Dynamics (training-free)](https://github.com/Zig-HS/D3)** (Zheng et al. · ICCV · 2025) — [code](https://github.com/Zig-HS/D3) · _in SynthScope: `d3`_ + Training-free: 2nd-order std of frozen per-frame features as the score. + _Results: GenVideo-Val ~98 mAP (paper). SynthScope faithful protocol (X-CLIP-B/16 + 10% edge-crop): AUROC 95.95 / acc 91.7 (the paper's mAP isn't comparable on SynthScope's 10000:56 real:fake pool — AUROC is the robust metric)._ +- **[ReStraV: Representation-Trajectory Geometry](https://github.com/ChristianInterno/ReStraV)** (Internò et al. · 2025) — [code](https://github.com/ChristianInterno/ReStraV) · _in SynthScope: `restrav`_ + Frozen DINOv2 trajectory-geometry features + a standardized MLP head. + _Results: GenVidBench Main ~93; SynthScope 92.7 mAP._ +- **[STALL: Spatio-Temporal whitened Log-Likelihoods (training-free)](https://github.com/OmerBenHayun/STALL)** (Ben Hayun et al. · CVPR · 2026) — [code](https://github.com/OmerBenHayun/STALL) · _in SynthScope: `stall`_ + Training-free spatial + temporal whitened-Gaussian log-likelihoods vs a real-video calibration. + _Results: GenVideo-Val ~0.80 AUC (paper); SynthScope AUROC 83.5._ +- **[ATSS: Anomalous Temporal Self-Similarity for AI-Generated Video Detection](https://github.com/hwang-cs-ime/ATSS)** (Wang et al. (hwang-cs-ime) · arXiv · 2026) — [code](https://github.com/hwang-cs-ime/ATSS) + Multimodal detector that exploits anomalous temporal self-similarity fingerprints across frames of AI-generated video. Inference code + a pretrained checkpoint (best.pt) are released; the data sampling/extraction pipeline is still being cleaned for release. + _Results: Claims SOTA in AP/AUC/ACC over GenVideo, EvalCrafter, VideoPhy, and VidProM vs prior methods (e.g. DeMamba). Repo live 2026-06-21 with code+weights; per-generator numbers not yet in the README. [integration pending: the released checkpoint's forward() consumes PRECOMPUTED multimodal temporal-self-similarity tensors (sim_img/sim_txt/sim_cross, BLIP2-based), not raw video — and the feature-extraction/data-sampling front-end that produces them is 'still being cleaned for release (coming soon)'. SynthScope feeds raw video clips, so there is no faithful way to run ATSS until that pipeline is published; will revisit when released. Re-checked 2026-10-05: unchanged.]_ +- **[G2VD: Generalizable AI-Generated Video Detection via Counterfactual Intervention and Causal Disentanglement](https://github.com/DMOSCAR-98/G2VD)** (Du et al. (dumeng98) · arXiv · 2026) — [code](https://github.com/DMOSCAR-98/G2VD) + Counterfactual-intervention + causal-disentanglement detector: a training-time CFIPipeline synthesizes controlled counterfactual samples (VAEs + frequency/pixel alignment), and a causal-disentanglement classifier with two domain-anchored branches and an HSIC independence constraint isolates the generalizable artifact signal from domain-specific content. + _Results: GenVidBench cross-domain: >90% accuracy / AUC ~0.95 using only 10% of the training data (paper). Repo live 2026-07-07 (CLIP/XCLIP/DeMamba backbones — the same family as SynthScope's D3). [integration pending (re-checked 2026-10-05): repo moved from dumeng98/G2VD (now 404) to DMOSCAR-98/G2VD; checkpoints are still NOT released ('not stored in this repository', MODEL_ZOO.md lists no links) and the CFIPipeline VAE-pool is part of the staged pipeline. Revisit when checkpoints appear.]_ +- **[RIFT: Mind the Rift - Cross-Scale Coupling Mismatch for AI-Generated Video Detection](https://github.com/Litsay/RIFT)** (Litsay · arXiv (submitted ACM MM'26) · 2026) — [code](https://github.com/Litsay/RIFT) + Treats a cross-scale coupling mismatch between macro temporal dynamics (differential geometry / persistent homology on frozen DINOv2 features) and micro residuals (steganalytic filtering + RAFT motion) as the forensic signal, scored by a coupling-divergence module. + _Results: Own benchmark: 99.22% acc / 99.33 F1 / 99.96 AUC, 97.87% leave-one-generator-out (paper; no head-to-head against named prior detectors in the abstract, so not directly comparable). [integration pending (assessed 2026-10-02): trained weights are 'released upon acceptance' (not available), and it needs DINOv2 + RAFT features over ~120K training videos. Re-checked 2026-10-05: unchanged.]_ + +## Benchmarks & curated lists +- [GenImage: million-scale real-vs-fake benchmark](https://github.com/GenImage-Dataset/GenImage) — ImageNet-1k reals + ~equal synthetic from diverse generators. +- [AIGIBench: robustness/generalization benchmark (23 fake subsets)](https://github.com/HorizonTEL/AIGIBench) — Multi-source generalization, degradation robustness, augmentation sensitivity, preprocessing impact. +- [AI-GenBench: temporal (GAN→diffusion) generalization benchmark](https://github.com/MI-BioLab/AI-GenBench) — Incremental, historically-ordered training to test generalization to new generators (Verimedia @ IJCNN 2025). +- [GenVidBench: 6M AI-generated video detection benchmark](https://arxiv.org/abs/2501.11340) — Large-scale benchmark for AI-generated video detection. +- [DeepfakeBench: comprehensive deepfake-detection benchmark](https://github.com/SCLBD/DeepfakeBench) — Unified benchmark across many deepfake datasets/detectors. +- [Awesome-AIGC-Image-Video-Detection (curated list)](https://github.com/ant-research/Awesome-AIGC-Image-Video-Detection) — Continuously-updated collection of image + video AIGC-detection papers and code. +- [Awesome-Comprehensive-Deepfake-Detection (curated list)](https://github.com/qiqitao77/Awesome-Comprehensive-Deepfake-Detection) — Broad survey-style index of deepfake-detection research and repos. + +--- +*Auto-generated from `interface/backend/trending.json` by `repro/generate_trending_readme.py` — edit the JSON, not this file.* diff --git a/_synthscope_env.py b/_synthscope_env.py new file mode 100644 index 0000000000000000000000000000000000000000..4d4b2493cd97c7994ae2bd634c4e0dbba9a84062 --- /dev/null +++ b/_synthscope_env.py @@ -0,0 +1,19 @@ +"""Backward compatibility for the pre-rename ``AGID_*`` environment variables. + +The project was renamed from AGID to SynthScope; every setting is now ``SYNTHSCOPE_``. Old launch scripts that still +export ``AGID_`` keep working: importing this module copies each such variable to its new name unless the new +name is already set. Dependency-free and idempotent, so any entry point can ``import _synthscope_env`` first. +""" +import os + +LEGACY_PREFIX = "AGID_" +PREFIX = "SYNTHSCOPE_" + + +def apply_legacy_env() -> None: + for key, value in list(os.environ.items()): + if key.startswith(LEGACY_PREFIX): + os.environ.setdefault(PREFIX + key[len(LEGACY_PREFIX):], value) + + +apply_legacy_env() diff --git a/accelerate_config.yaml b/accelerate_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e583046363d2ce5ba0179e120e308dfce74892b --- /dev/null +++ b/accelerate_config.yaml @@ -0,0 +1,19 @@ +compute_environment: LOCAL_MACHINE +distributed_type: DEEPSPEED +mixed_precision: "no" +num_machines: 1 +num_processes: 4 # number of GPUs +machine_rank: 0 +main_training_function: main +rdzv_backend: static +same_network: true +use_cpu: false + +deepspeed_config: + deepspeed_multinode_launcher: standard + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: false + zero_stage: 2 \ No newline at end of file diff --git a/agid/__init__.py b/agid/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0890867349bcedd251c20ab2a2a32a88a25285cc --- /dev/null +++ b/agid/__init__.py @@ -0,0 +1,13 @@ +"""Deprecated alias of :mod:`synthscope` (the project was renamed from AGID to SynthScope). + +``import agid`` keeps working and returns the very same module object, so ``agid.load(...)``, +``from agid.api import Detector`` and ``python -m agid`` behave exactly as before. +""" +import sys + +import synthscope +from synthscope import api, cli, hub + +sys.modules[__name__] = synthscope +for _name, _mod in (("api", api), ("cli", cli), ("hub", hub)): + sys.modules[f"{__name__}.{_name}"] = _mod diff --git a/agid/__main__.py b/agid/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..16feed9ea5ddb9b206874387c78aa7041ad081d0 --- /dev/null +++ b/agid/__main__.py @@ -0,0 +1,5 @@ +"""``python -m agid`` -> same as ``python -m synthscope`` (deprecated alias).""" +from synthscope.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7839a61f77ff696b990c3c3accd9b15826d2d267 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,82 @@ +# SynthScope Benchmark & Leaderboard (v1) + +> Looking for the full setup guide (datasets, paper reproduction, generator benchmarks, run times)? See [`docs/BENCHMARKS.md`](../docs/BENCHMARKS.md). +> This page covers the standardized, versioned submission track. + +A standardized, **versioned** evaluation protocol for AI-generated image & video detection, so that +methods are comparable under identical data, metrics, and scoring. The leaderboard is generated +from the submission files in [`submissions/`](submissions/) — see [LEADERBOARD.md](../LEADERBOARD.md). + +> **Why:** every detection paper ships its own repo, data format, and metric conventions, which +> makes cross-method comparison unreliable. SynthScope fixes the protocol once and evaluates every +> detector through the same pipeline. + +## Tracks (v1) + +| Track id | Modality | Test set | Train signal | Primary metric | +| --- | --- | --- | --- | --- | +| `forensynths` | image | ForenSynths real/fake, scored **per generator** | ProGAN-trained (cross-generator generalization) | **mAP** (mean AP over generators) | +| `genimage-sdv1.4` | image | GenImage SD-v1.4 real/fake | in-distribution diffusion | **AUROC** | +| `genvideo-val` | video | GenVideo validation split | mixed | **AP** | + +**These three tracks are only the start.** The repo supports 13 image and 8 video benchmark datasets (see the table in the +[README](../README.md#-benchmarks)), and the web page's Leaderboard tab already compares detectors on 30+ benchmark groups. +A dataset becomes a standardized track once its protocol (test set, grouping, metric) is fixed, so results stay comparable; +adding one means a schema entry, an `evaluate.py` track and verified submissions, described in +[`docs/BENCHMARKS.md`](../docs/BENCHMARKS.md). The datasets themselves download and organize themselves. + +Metrics are reported in **%**. `mAP` = mean Average Precision over generators; `AUROC`/`AP` use the +detector's native fake-probability; `Acc` = mean balanced accuracy (mean of real- and fake-accuracy) +over generators at threshold 0.5. + +## Protocol rules + +1. **No test-set training or tuning.** Thresholds/calibration must be fixed before evaluation. +2. **Report per generator, then average.** A track's headline number is the unweighted mean over + generators (not a pooled number), which measures generalization rather than rewarding a few + large generators. +3. **Declare provenance** via `evaluation`: `official-weights`, `from-scratch`, `frozen-probe`, + `training-free`, or `external` (numbers produced outside SynthScope). +4. **Verified numbers trace to a run.** `verified: true` means the result came from a logged SynthScope + evaluation (set `source` to the log/run path). Reference seed entries are transcribed from the + repo's reproduction table. + +## Data layout + +The evaluator expects the SynthScope common layout — one directory per generator, each split into +`real/` and `fake/`: + +``` +/ + /real/*.png /fake/*.png + /real/*.png /fake/*.png +``` + +A single pooled set is just one generator directory (or `/real`, `/fake`). + +## Evaluate a detector and produce a submission + +```bash +python benchmark/evaluate.py \ + --model fat_former --track forensynths \ + --data /path/to/forensynths_test \ + --out benchmark/submissions/fat_former.json +``` + +This loads the detector via the `synthscope` API, scores every image (batched), computes per-generator + +mean metrics, and writes a schema-valid submission JSON. Then regenerate the table: + +```bash +python benchmark/leaderboard.py # writes LEADERBOARD.md +python benchmark/leaderboard.py --check # CI: fail if LEADERBOARD.md is stale or a submission is invalid +``` + +## Submit a result (PR) + +1. Add one JSON file under `submissions/` (see [`schema.json`](schema.json) and existing entries). +2. Run `python benchmark/leaderboard.py` and commit the updated `LEADERBOARD.md`. +3. Open a PR. CI validates your submission and that the table is up to date. Prefer `verified` + results produced by `evaluate.py`; for `external` numbers, link the source. + +Versioning: this is **v1**. Protocol-breaking changes bump to v2 and start a new leaderboard so old +numbers stay meaningful. diff --git a/benchmark/aggregate_results.py b/benchmark/aggregate_results.py new file mode 100644 index 0000000000000000000000000000000000000000..94cd30a57b914516e6e49823f00d893fe8dd8700 --- /dev/null +++ b/benchmark/aggregate_results.py @@ -0,0 +1,102 @@ +"""Aggregate the self-recorded bench_eval results into a master reproduced-results table. + +Reads every repro/bench_results/.json (latest run per tag), prints a compact summary, writes +repro/RESULTS_TABLE.md (model | weights | benchmark | n_gen | acc / mAP / mAUROC), and supports +computing a paper-matching subset mean from a run's per-generator dict: + + python benchmark/aggregate_results.py # refresh RESULTS_TABLE.md + python benchmark/aggregate_results.py --subset fatformer_official_forensynths progan,stylegan,... + python benchmark/aggregate_results.py --detail npr_official_diffusionforensics +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +REPRO = Path(__file__).resolve().parents[1] / "repro" +RESULTS_DIR = REPRO / "bench_results" + + +def load_runs() -> dict[str, dict]: + runs: dict[str, dict] = {} + for p in sorted(RESULTS_DIR.glob("*.json")): + if p.stem.startswith(("_validate", "_probe", "_detail")): + continue + try: + d = json.loads(p.read_text()) + except Exception as e: # noqa: BLE001 + print(f" skip {p.name}: {e}") + continue + tag = d.get("tag", p.stem) + if tag not in runs or d.get("date", "") >= runs[tag].get("date", ""): + runs[tag] = d + return runs + + +def weights_of(tag: str) -> str: + if "_official_" in tag: + return "official" + if "_synthscope_" in tag: + return "synthscope" + return "other" + + +def subset_mean(run: dict, gens: list[str]) -> dict: + pg = run.get("per_generator", {}) + found = [g for g in gens if g in pg] + missing = [g for g in gens if g not in pg] + if not found: + return {"error": f"none of {gens} in {list(pg)}"} + def m(k): + return round(sum(pg[g][k] for g in found) / len(found), 2) + return {"n": len(found), "gens": found, "missing": missing, + "acc": m("acc"), "map": m("ap"), "mauroc": m("auroc")} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--subset", nargs=2, metavar=("TAG", "GENS"), + help="compute mean over a comma list of generators for one run tag") + ap.add_argument("--detail", metavar="TAG", help="print per-generator detail for one run tag") + args = ap.parse_args() + runs = load_runs() + + if args.subset: + tag, gens = args.subset + if tag not in runs: + print(f"no run '{tag}'. have: {sorted(runs)}"); return 1 + print(json.dumps(subset_mean(runs[tag], gens.split(",")), indent=2)); return 0 + + if args.detail: + if args.detail not in runs: + print(f"no run '{args.detail}'. have: {sorted(runs)}"); return 1 + r = runs[args.detail] + print(f"# {args.detail} ({r['model']}, {r['benchmark']}, {r.get('n_generators')} gens) means={r['means']}") + for g, m in sorted(r["per_generator"].items()): + print(f" {g:32s} acc={m['acc']:6.2f} AP={m['ap']:6.2f} AUROC={m['auroc']:6.2f} " + f"(real={m['real_acc']:.1f} fake={m['fake_acc']:.1f})") + return 0 + + # default: master table + rows = [] + for tag, r in sorted(runs.items()): + me = r.get("means", {}) + rows.append((r.get("model", "?"), weights_of(tag), r.get("benchmark", "?"), + tag, r.get("n_generators", 0), + me.get("acc"), me.get("map"), me.get("mauroc"))) + rows.sort(key=lambda x: (x[0], x[1], x[3])) + lines = ["# SynthScope reproduced benchmark results (auto-generated from repro/bench_results/)\n", + "| model | weights | benchmark | tag | n_gen | acc | mAP | mAUROC |", + "| --- | --- | --- | --- | --- | --- | --- | --- |"] + for model, w, bench, tag, n, acc, mp, au in rows: + lines.append(f"| {model} | {w} | {bench} | `{tag}` | {n} | {acc} | {mp} | {au} |") + out = REPRO / "RESULTS_TABLE.md" + out.write_text("\n".join(lines) + "\n") + print("\n".join(lines)) + print(f"\nWrote {out} ({len(rows)} runs)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/bench_eval.py b/benchmark/bench_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..5e7d21d367d7030e1021e33d9bb2302aab33f507 --- /dev/null +++ b/benchmark/bench_eval.py @@ -0,0 +1,305 @@ +"""Unified benchmark eval for SynthScope detectors — reproduce a paper's per-generator table. + +Scores a detector over a benchmark's per-generator real/fake folders through the SAME verified +synthscope inference path the web app uses (engine + registry + per-model preprocessing + fake-prob), +computes per-generator + mean acc@0.5 / AP / AUROC, writes a JSON, and APPENDS a verified line to +``repro/MASTER_RESULTS.jsonl`` so every reported number traces to this run + its job log. + + python benchmark/bench_eval.py --model clip --benchmark forensynths \ + --data /hddstore/ssalehi/SynthScope/CNNDetection/test --tag clip_synthscope_forensynths + + # one step: download the dataset if needed (its built-in loader arranges it), pick the profile, evaluate + python benchmark/bench_eval.py --model npr --dataset commgen15 --root /data/CommGen15 + +Benchmark profiles encode each dataset's correct generator grouping + real-pairing: + forensynths CNNDetection/test per top-level generator; reals+fakes collected recursively + (handles progan//{0_real,1_fake} nesting) + ufd UniversalFakeDetect fakes per generator; reals paired Ojha-style + (guided->imagenet, ldm/glide/dalle->laion) + pergen freqnet/aigibench/ each /{0_real,1_fake}; optional shared --reals + cospybench/generic + pool GenImage/single-pool one group from /{real,fake} or {0_real,1_fake} + +Use --ckpt to override the checkpoint (official vs SynthScope-trained, for side-by-side runs). +""" +from __future__ import annotations + +import argparse +import datetime +import json +from pathlib import Path + +import torch +from PIL import Image, ImageFile + +import synthscope +from models.image.cure.preprocess import SRC_FORMAT_KEY +from datasets_util import ensure_dataset, find_eval_root + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff", ".JPEG"} +REAL_DIRNAMES = ("0_real", "real", "reals") +FAKE_DIRNAMES = ("1_fake", "fake", "fakes", "ai") +SHARED_REAL_DIRS = {"imagenet", "laion", "_real_pool", "real", "0_real", "reals"} +RESULTS_LOG = Path(__file__).resolve().parents[1] / "repro" / "MASTER_RESULTS.jsonl" +# Ojha UniversalFakeDetect real pairing (verified against the official dataset_paths.py). +UFD_REAL = {"guided": "imagenet"} # everything else -> laion +UFD_DEFAULT_REAL = "laion" + + +def _images(d: Path) -> list[Path]: + if not d or not d.is_dir(): + return [] + return sorted(p for p in d.rglob("*") if p.suffix.lower() in {e.lower() for e in IMAGE_EXTS}) + + +def _collect(gen_dir: Path, dirnames: tuple[str, ...]) -> list[Path]: + """All images under any subdir of gen_dir whose name matches one of dirnames (any depth).""" + out: list[Path] = [] + for name in dirnames: + for sub in gen_dir.rglob(name): + if sub.is_dir(): + out += [p for p in sub.rglob("*") if p.suffix.lower() in {e.lower() for e in IMAGE_EXTS}] + # de-dup while keeping order + seen, uniq = set(), [] + for p in out: + if p not in seen: + seen.add(p); uniq.append(p) + return sorted(uniq) + + +def discover(benchmark: str, root: Path, shared_reals: Path | None, select: set[str] | None): + """Return [(generator, real_paths, fake_paths)] for the benchmark.""" + shared = _images(shared_reals) if shared_reals else [] + + if benchmark == "pool": + reals = _collect(root, REAL_DIRNAMES) or shared + fakes = _collect(root, FAKE_DIRNAMES) + return [("all", reals, fakes)] if reals and fakes else [] + + if benchmark == "ufd": + out = [] + gens = sorted(p.name for p in root.iterdir() if p.is_dir() and p.name not in SHARED_REAL_DIRS) + for gen in gens: + if select and gen not in select: + continue + real_src = root / UFD_REAL.get(gen, UFD_DEFAULT_REAL) + reals = _collect(real_src, REAL_DIRNAMES) + fakes = _collect(root / gen, FAKE_DIRNAMES) + if reals and fakes: + out.append((gen, reals, fakes)) + else: + print(f" skip {gen}: reals={len(reals)} fakes={len(fakes)}") + return out + + if benchmark == "forensynths": + out = [] + for gen_dir in sorted(p for p in root.iterdir() if p.is_dir()): + if select and gen_dir.name not in select: + continue + reals = _collect(gen_dir, REAL_DIRNAMES) or shared + fakes = _collect(gen_dir, FAKE_DIRNAMES) + if reals and fakes: + out.append((gen_dir.name, reals, fakes)) + else: + print(f" skip {gen_dir.name}: reals={len(reals)} fakes={len(fakes)}") + return out + + # pergen / generic: each top-level dir (excluding shared-real dirs) is a generator + out = [] + for gen_dir in sorted(p for p in root.iterdir() if p.is_dir()): + if gen_dir.name in SHARED_REAL_DIRS and gen_dir.name != root.name: + continue + if select and gen_dir.name not in select: + continue + reals = _collect(gen_dir, REAL_DIRNAMES) or shared + fakes = _collect(gen_dir, FAKE_DIRNAMES) + if reals and fakes: + out.append((gen_dir.name, reals, fakes)) + else: + print(f" skip {gen_dir.name}: reals={len(reals)} fakes={len(fakes)}") + return out + + +def native_res_transform(tf): + """Drop the up-scaling Resize/CenterCrop from a model's eval transform, keeping + ToTensor + Normalize, so images are scored at NATIVE resolution. + + Some methods (e.g. NPR on GANGen-Detection/freqnet) detect up-sampling artifacts and + MUST NOT be fed a bilinearly up-scaled image — resizing a 128x128 GAN crop to 256x256 + injects the exact artifact they key on, collapsing the real class. The official NPR + GANGen recipe runs with ``no_resize=True``; this reproduces it. Safe only when every + image in a per-generator batch shares one resolution (true for freqnet's per-gen sets), + since bench_eval stacks a batch into a single tensor.""" + import torchvision.transforms as T + if not isinstance(tf, T.Compose): + return tf + kept = [t for t in tf.transforms if not isinstance(t, (T.Resize, T.CenterCrop))] + return T.Compose(kept) + + +@torch.no_grad() +def _open_rgb(path: Path) -> Image.Image: + """Open ``path`` as RGB, recording the source file format in ``img.info`` so format-aware eval + transforms (CuRe's PNG->JPEG96 step) see what the official loader sees.""" + source = Image.open(path) + source_format = source.format + image = source.convert("RGB") + image.info[SRC_FORMAT_KEY] = source_format + return image + + +def score(det, model, spec, paths: list[Path], bs: int, transform=None) -> torch.Tensor: + device = det._engine.device + tf = transform if transform is not None else spec.transform + # Native-resolution specs (SPAI any-res) produce variable-size tensors that cannot be stacked, + # so they MUST be scored one image at a time. + if getattr(spec, "native_res", False): + bs = 1 + probs = [] + for i in range(0, len(paths), bs): + batch = [] + for p in paths[i:i + bs]: + try: + batch.append(tf(_open_rgb(p))) + except Exception as e: # noqa: BLE001 + print(f" bad image {p}: {e}") + if not batch: + continue + x = torch.stack(batch).to(device) + probs.append(det._engine._fake_prob(model, spec, x).detach().float().cpu()) + return torch.cat(probs) if probs else torch.empty(0) + + +def metrics(fake_probs: torch.Tensor, labels: torch.Tensor) -> dict | None: + from torchmetrics.functional import auroc, average_precision + if labels.unique().numel() < 2: + return None + ap = float(average_precision(fake_probs, labels, task="binary")) * 100 + au = float(auroc(fake_probs, labels, task="binary")) * 100 + preds = (fake_probs >= 0.5).int() + real_acc = float((preds[labels == 0] == 0).float().mean()) * 100 + fake_acc = float((preds[labels == 1] == 1).float().mean()) * 100 + return {"ap": round(ap, 2), "auroc": round(au, 2), + "real_acc": round(real_acc, 2), "fake_acc": round(fake_acc, 2), + "acc": round((real_acc + fake_acc) / 2, 2)} + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("-m", "--model", required=True, help="SynthScope detector key (see `synthscope list`)") + ap.add_argument("--benchmark", choices=["forensynths", "ufd", "pergen", "pool"], + help="how the folders are grouped (chosen automatically with --dataset)") + ap.add_argument("--data", help="benchmark root (not needed with --dataset)") + ap.add_argument("--dataset", help="a supported dataset (see `python benchmark/download_dataset.py --list`): downloads it " + "if missing, finds the folder to evaluate and the right --benchmark profile") + ap.add_argument("--root", help="with --dataset: where the dataset lives / should be downloaded (default: root_dir in its yaml)") + ap.add_argument("--no-download", action="store_true", help="with --dataset: fail instead of downloading missing data") + ap.add_argument("--dry-run", action="store_true", help="with --dataset: only show the resolved folder and profile") + ap.add_argument("--reals", default=None, help="shared real dir (pergen/forensynths fallback)") + ap.add_argument("--ckpt", default=None, help="checkpoint override (official vs SynthScope weights)") + ap.add_argument("--select", default=None, help="comma list: only these generators") + ap.add_argument("--limit", type=int, default=None, help="cap images per class per generator") + ap.add_argument("--bs", type=int, default=32) + ap.add_argument("--native-res", action="store_true", + help="score at native resolution (strip up-scaling Resize/CenterCrop); " + "for artifact-based methods like NPR on low-res GAN sets (freqnet)") + ap.add_argument("--device", default=None) + ap.add_argument("--tag", help="run tag (output filename + log); default _") + ap.add_argument("--out", default=None) + args = ap.parse_args(argv) + + if args.dataset: + root = ensure_dataset(args.dataset, "image", args.root, download=not args.no_download and not args.dry_run) + eval_root, profile = find_eval_root(args.dataset, root) + args.data = args.data or str(eval_root) + args.benchmark = args.benchmark or profile + args.tag = args.tag or f"{args.model}_{args.dataset}" + print(f"[dataset] {args.dataset}: evaluating {args.data} with the '{args.benchmark}' profile", flush=True) + if args.dry_run: + return 0 + if not (args.benchmark and args.data): + ap.error("give --dataset NAME, or both --benchmark and --data") + if not args.tag: + ap.error("--tag is required without --dataset") + + load_kw = {} + if args.ckpt: + load_kw["checkpoint"] = args.ckpt + det = synthscope.load(args.model, device=args.device, **load_kw) + det._ensure_weights() + spec = det.spec + model = det._engine._get_model(spec) + eval_transform = native_res_transform(spec.transform) if args.native_res else spec.transform + if args.native_res: + print(f"[native-res] scoring at native resolution; transform={eval_transform}", flush=True) + + root = Path(args.data).expanduser() + select = set(args.select.split(",")) if args.select else None + gens = discover(args.benchmark, root, + Path(args.reals).expanduser() if args.reals else None, select) + if not gens: + print(f"No generators with real+fake under {root} (benchmark={args.benchmark})") + return 1 + + print(f"=== bench_eval {args.tag} | model={args.model} ckpt={args.ckpt or 'registry-default'} " + f"| {args.benchmark} | {len(gens)} generators | device={det._engine.device} ===", flush=True) + + per_gen, agg = {}, {"ap": [], "auroc": [], "acc": []} + for name, reals, fakes in gens: + if args.limit: + reals, fakes = reals[:args.limit], fakes[:args.limit] + labels = torch.tensor([0] * len(reals) + [1] * len(fakes)) + print(f" {name}: {len(reals)} real + {len(fakes)} fake ...", flush=True) + probs = score(det, model, spec, reals + fakes, args.bs, transform=eval_transform) + if probs.numel() != labels.numel(): + labels = labels[:probs.numel()] + m = metrics(probs, labels) + if m is None: + print(f" skip {name}: one class only") + continue + per_gen[name] = m + for k in agg: + agg[k].append(m[k]) + print(f" {name}: acc={m['acc']} AP={m['ap']} AUROC={m['auroc']} " + f"(real_acc={m['real_acc']} fake_acc={m['fake_acc']})", flush=True) + + if not per_gen: + print("No generator produced valid metrics.") + return 1 + + def mean(xs): + return round(sum(xs) / len(xs), 2) + + means = {"acc": mean(agg["acc"]), "map": mean(agg["ap"]), "mauroc": mean(agg["auroc"])} + result = { + "tag": args.tag, + "model": args.model, + "ckpt": args.ckpt or "registry-default", + "benchmark": args.benchmark, + "data_root": str(root), + "n_generators": len(per_gen), + "means": means, + "per_generator": per_gen, + "date": datetime.datetime.now().isoformat(timespec="seconds"), + "synthscope_version": getattr(synthscope, "__version__", "?"), + } + out = Path(args.out).expanduser() if args.out else ( + RESULTS_LOG.parent / "bench_results" / f"{args.tag}.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(result, indent=2) + "\n") + + RESULTS_LOG.parent.mkdir(parents=True, exist_ok=True) + with RESULTS_LOG.open("a") as f: + f.write(json.dumps({k: result[k] for k in + ("date", "tag", "model", "ckpt", "benchmark", "data_root", + "n_generators", "means")}) + "\n") + + print(f"\n=== RESULT {args.tag}: mAP={means['map']} mAUROC={means['mauroc']} acc={means['acc']} " + f"over {len(per_gen)} generators -> {out} ===", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/bench_eval_video.py b/benchmark/bench_eval_video.py new file mode 100644 index 0000000000000000000000000000000000000000..38e9b8ff768f98d296c4c63d1d2a917237a3c382 --- /dev/null +++ b/benchmark/bench_eval_video.py @@ -0,0 +1,181 @@ +"""Unified VIDEO benchmark eval for SynthScope video detectors (d3, restrav, stall). + +Scores each video through the SAME verified synthscope engine path the web app uses +(engine.predict_video_clip -> native clip pipeline + per-model norm + fake-prob), computes +per-generator + mean acc@0.5 / AP / AUROC, and writes the SAME JSON schema as bench_eval.py +(repro/bench_results/.json + append repro/MASTER_RESULTS.jsonl) so the aggregator ingests it. + + python benchmark/bench_eval_video.py --model d3 --benchmark genvideo \ + --data /hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val --limit 150 --tag d3_genvideo + +Benchmark layouts: + genvideo /Fake//*.mp4 + /Real/*.mp4 (shared real pool) [GenVideo-Val] + fakeonly //*.mp4 (all fake) + --reals DIR of real videos [EvalCrafter] + pergen //{0_real,1_fake}/*.mp4 [GenVidBench] +""" +from __future__ import annotations + +import argparse +import datetime +import json +import sys +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "interface")) +from backend.engine import get_engine # noqa: E402 +from backend.registry import get_spec # noqa: E402 + +VIDEO_EXTS = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".gif", ".m4v"} +REPRO = Path(__file__).resolve().parents[1] / "repro" +RESULTS_LOG = REPRO / "MASTER_RESULTS.jsonl" +REAL_DIRNAMES = ("0_real", "real", "reals", "Real") +FAKE_DIRNAMES = ("1_fake", "fake", "fakes", "Fake") + + +def _videos(d: Path) -> list[Path]: + if not d or not d.is_dir(): + return [] + return sorted(p for p in d.rglob("*") if p.suffix.lower() in VIDEO_EXTS) + + +def _find(parent: Path, names) -> Path | None: + for n in names: + if (parent / n).is_dir(): + return parent / n + return None + + +def discover(benchmark: str, root: Path, shared_reals: Path | None, select): + shared = _videos(shared_reals) if shared_reals else [] + out = [] + if benchmark == "genvideo": + fake_root = _find(root, FAKE_DIRNAMES) + real_root = _find(root, REAL_DIRNAMES) + reals = _videos(real_root) or shared + gens = sorted(p for p in fake_root.iterdir() if p.is_dir()) if fake_root else [] + for g in gens: + if select and g.name not in select: + continue + fk = _videos(g) + if fk and reals: + out.append((g.name, reals, fk)) + return out + if benchmark == "fakeonly": + gens = sorted(p for p in root.iterdir() if p.is_dir()) + for g in gens: + if select and g.name not in select: + continue + fk = _videos(g) + if fk and shared: + out.append((g.name, shared, fk)) + return out + # pergen + for g in sorted(p for p in root.iterdir() if p.is_dir()): + if select and g.name not in select: + continue + reals = _videos(_find(g, REAL_DIRNAMES)) or shared + fakes = _videos(_find(g, FAKE_DIRNAMES)) + if reals and fakes: + out.append((g.name, reals, fakes)) + return out + + +def metrics(fake_probs: torch.Tensor, labels: torch.Tensor): + from torchmetrics.functional import auroc, average_precision + if labels.unique().numel() < 2: + return None + ap = float(average_precision(fake_probs, labels, task="binary")) * 100 + au = float(auroc(fake_probs, labels, task="binary")) * 100 + preds = (fake_probs >= 0.5).int() + real_acc = float((preds[labels == 0] == 0).float().mean()) * 100 + fake_acc = float((preds[labels == 1] == 1).float().mean()) * 100 + return {"ap": round(ap, 2), "auroc": round(au, 2), "real_acc": round(real_acc, 2), + "fake_acc": round(fake_acc, 2), "acc": round((real_acc + fake_acc) / 2, 2)} + + +@torch.no_grad() +def score_videos(engine, spec, paths): + probs = [] + for p in paths: + try: + probs.append(engine.predict_video_clip(spec, str(p))["fake_probability"]) + except Exception as e: # noqa: BLE001 + print(f" bad video {p.name}: {e}") + return probs + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("-m", "--model", required=True, choices=["d3", "restrav", "stall"]) + ap.add_argument("--benchmark", required=True, choices=["genvideo", "fakeonly", "pergen"]) + ap.add_argument("--data", required=True) + ap.add_argument("--reals", default=None) + ap.add_argument("--select", default=None) + ap.add_argument("--limit", type=int, default=None, help="cap videos per class per generator") + ap.add_argument("--tag", required=True) + ap.add_argument("--out", default=None) + args = ap.parse_args(argv) + + engine = get_engine() + spec = get_spec(args.model) + engine._get_model(spec) + root = Path(args.data) + select = set(args.select.split(",")) if args.select else None + gens = discover(args.benchmark, root, Path(args.reals) if args.reals else None, select) + if not gens: + print(f"No generators with real+fake videos under {root} ({args.benchmark})") + return 1 + print(f"=== bench_eval_video {args.tag} | {args.model} | {args.benchmark} | {len(gens)} gens " + f"| device={engine.device} ===", flush=True) + + per_gen, agg = {}, {"ap": [], "auroc": [], "acc": []} + # score the shared real pool once if it is identical across generators + real_cache = {} + for name, reals, fakes in gens: + if args.limit: + reals, fakes = reals[:args.limit], fakes[:args.limit] + rk = (id(reals) if args.limit is None else None) + print(f" {name}: {len(reals)} real + {len(fakes)} fake ...", flush=True) + key = tuple(reals[:1]) + (len(reals),) + if key in real_cache: + rp = real_cache[key] + else: + rp = score_videos(engine, spec, reals) + real_cache[key] = rp + fp = score_videos(engine, spec, fakes) + probs = torch.tensor(rp + fp) + labels = torch.tensor([0] * len(rp) + [1] * len(fp)) + m = metrics(probs, labels) + if m is None: + print(f" skip {name}: one class"); continue + per_gen[name] = m + for k in agg: + agg[k].append(m[k]) + print(f" {name}: acc={m['acc']} AP={m['ap']} AUROC={m['auroc']}", flush=True) + + if not per_gen: + print("No generator produced metrics."); return 1 + + def mean(xs): + return round(sum(xs) / len(xs), 2) + means = {"acc": mean(agg["acc"]), "map": mean(agg["ap"]), "mauroc": mean(agg["auroc"])} + result = {"tag": args.tag, "model": args.model, "ckpt": "registry-default", + "benchmark": f"video:{args.benchmark}", "data_root": str(root), + "n_generators": len(per_gen), "means": means, "per_generator": per_gen, + "date": datetime.datetime.now().isoformat(timespec="seconds"), + "synthscope_version": "video"} + out = Path(args.out) if args.out else (REPRO / "bench_results" / f"{args.tag}.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(result, indent=2) + "\n") + with RESULTS_LOG.open("a") as f: + f.write(json.dumps({k: result[k] for k in + ("date", "tag", "model", "ckpt", "benchmark", "data_root", "n_generators", "means")}) + "\n") + print(f"\n=== RESULT {args.tag}: mAP={means['map']} mAUROC={means['mauroc']} acc={means['acc']} " + f"over {len(per_gen)} gens -> {out} ===", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/datasets_util.py b/benchmark/datasets_util.py new file mode 100644 index 0000000000000000000000000000000000000000..4e0817d01cd06e072d59e5851c108e03ba9e87a4 --- /dev/null +++ b/benchmark/datasets_util.py @@ -0,0 +1,99 @@ +"""Shared helpers for the benchmark scripts: download a dataset with its built-in loader and find where to evaluate it. + +``ensure_dataset`` composes the dataset's Hydra config (``conf/dataset//.yaml``), and, if the data is not +on disk yet, constructs its dataset module, whose loader (``prefetch: true``) downloads the data and restructures it into +``/{0_real,1_fake}``. ``find_eval_root`` then locates the folder to evaluate and picks the ``bench_eval.py`` +profile (forensynths / ufd / pergen / pool) from the layout. +""" +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"} +REAL_NAMES = {"0_real", "real", "reals"} +FAKE_NAMES = {"1_fake", "fake", "fakes", "ai"} +# datasets whose generator grouping / real pairing is special-cased by a bench_eval profile +PROFILE_OVERRIDE = {"cnndetection": "forensynths", "cospy_subset": "forensynths", "spai_ldm": "forensynths", + "universialfakedetection": "ufd"} + + +def dataset_config(name: str, modality: str = "image", root: str | None = None): + """The composed Hydra config for ``conf/dataset//.yaml`` (``root`` overrides ``root_dir``).""" + from config_utils import compose_config + conf = ROOT / "conf" / "dataset" + if not (conf / modality / f"{name}.yaml").is_file(): + other = [m for m in ("image", "video") if (conf / m / f"{name}.yaml").is_file()] + if other: + raise SystemExit(f"{name} is a {other[0]} dataset; use the {other[0]} evaluator " + f"({'benchmark/bench_eval_video.py' if other[0] == 'video' else 'benchmark/bench_eval.py'})") + known = sorted(p.stem for p in (conf / modality).glob("*.yaml")) + raise SystemExit(f"unknown {modality} dataset '{name}'. Available: {', '.join(known)}") + overrides = [f"dataset@DatasetConfig={modality}/{name}"] + if root: + overrides.append(f"DatasetConfig.root_dir={root}") + return compose_config(overrides) + + +def has_images(root: Path) -> bool: + """True as soon as one image file is found under ``root`` (cheap: stops at the first hit).""" + if not root.is_dir(): + return False + for _, _, files in os.walk(root): + if any(os.path.splitext(f)[1].lower() in IMAGE_EXT for f in files): + return True + return False + + +def download_dataset(name: str, modality: str = "image", root: str | None = None) -> Path: + """Run the dataset's loader (download + restructure). Returns the root directory.""" + from dataset import AIGIDatasetModule + cfg = dataset_config(name, modality, root) + AIGIDatasetModule(cfg) # constructing the datasets runs the loader (prefetch: true downloads + restructures) + return Path(cfg.DatasetConfig.root_dir) + + +def ensure_dataset(name: str, modality: str = "image", root: str | None = None, download: bool = True) -> Path: + """The dataset's root directory, downloading it first when no images are there yet.""" + cfg = dataset_config(name, modality, root) + path = Path(cfg.DatasetConfig.root_dir).expanduser() + if has_images(path): + return path + if not download: + raise SystemExit(f"{name}: no data under {path} (drop --no-download to fetch it, or pass --root)") + print(f"[dataset] {name}: nothing under {path}, downloading with the built-in loader ...", flush=True) + return download_dataset(name, modality, root) + + +def _subdirs(d: Path) -> list[Path]: + try: + return sorted(c for c in d.iterdir() if c.is_dir() and not c.name.startswith((".", "_"))) + except OSError: + return [] + + +def _has_real_fake(d: Path, depth: int = 3) -> bool: + names = {c.name.lower() for c in d.iterdir() if c.is_dir()} if d.is_dir() else set() + if names & REAL_NAMES and names & FAKE_NAMES: + return True + return depth > 0 and any(_has_real_fake(c, depth - 1) for c in _subdirs(d)[:60]) + + +def find_eval_root(name: str, root: Path) -> tuple[Path, str]: + """(folder to evaluate, bench_eval profile). Looks in ``root/test``, ``root/test_full`` and ``root`` itself.""" + if name in PROFILE_OVERRIDE: # fixed layouts handled by their own bench_eval profile (e.g. UFD: fakes and reals in separate folders) + for cand in (root / "test", root): + if cand.is_dir() and _subdirs(cand): + return cand, PROFILE_OVERRIDE[name] + for cand in (root / "test", root / "test_full", root): + if not cand.is_dir(): + continue + names = {c.name.lower() for c in cand.iterdir() if c.is_dir()} + if names & REAL_NAMES and names & FAKE_NAMES: # one pooled real/fake set + return cand, PROFILE_OVERRIDE.get(name, "pool") + if any(_has_real_fake(g) for g in _subdirs(cand)): # one folder per generator + return cand, PROFILE_OVERRIDE.get(name, "pergen") + raise SystemExit(f"{name}: could not find real/fake folders under {root}; expected " + "/[test/]/{0_real,1_fake} (see docs/BENCHMARKS.md)") diff --git a/benchmark/download_dataset.py b/benchmark/download_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..639a9e7787e7a404713925a67767f3fa46ca9160 --- /dev/null +++ b/benchmark/download_dataset.py @@ -0,0 +1,60 @@ +"""Download and prepare one of SynthScope's benchmark datasets with its built-in loader. + + python benchmark/download_dataset.py --list # datasets and where each comes from + python benchmark/download_dataset.py genimage --root /data/GenImage + python benchmark/download_dataset.py synthbuster --root /data/Synthbuster --dry-run # only show what would happen + +The loaders download from the dataset's official source (Hugging Face, Zenodo, ModelScope or Google Drive; see +``conf/dataset/image/.yaml``) and restructure it into ``//{0_real,1_fake}``, which +``benchmark/bench_eval.py`` reads. Some hosts rate-limit (Google Drive) or ask you to accept terms first; if a download +fails, fetch the data by hand from the ``source_url`` and unpack it into ``--root`` in the same layout. +""" +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from datasets_util import dataset_config, download_dataset + +SOURCE_KEYS = ("source_url", "hf_repo_id", "zenodo_record", "repo_id", "modelscope_id", "setting") + + +def available() -> list[Path]: + return sorted((ROOT / "conf" / "dataset").glob("*/*.yaml")) + + +def describe(path: Path) -> str: + """One line: modality/name and where the data comes from.""" + from omegaconf import OmegaConf + cfg = OmegaConf.load(path) + src = ", ".join(f"{k}={cfg[k]}" for k in SOURCE_KEYS if k in cfg) + return f"{path.parent.name}/{path.stem:24s} {src or '(see the yaml)'}" + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("name", nargs="?", help="dataset config name, e.g. genimage, cnndetection, aigibench, synthbuster") + ap.add_argument("--root", help="where to put the data (default: root_dir from the dataset's yaml)") + ap.add_argument("--modality", default="image", choices=["image", "video"]) + ap.add_argument("--list", action="store_true", help="list the available datasets and their sources") + ap.add_argument("--dry-run", action="store_true", help="only print the resolved config; download nothing") + args = ap.parse_args() + + if args.list or not args.name: + print("\n".join(describe(p) for p in available())) + return + dc = dataset_config(args.name, args.modality, args.root).DatasetConfig + print(f"dataset {dc.name}: root_dir={dc.root_dir} prefetch={dc.get('prefetch')}") + for k in SOURCE_KEYS: + if k in dc: + print(f" {k}: {dc[k]}") + if args.dry_run: + return + path = download_dataset(args.name, args.modality, args.root) + print(f"done: {path}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/evaluate.py b/benchmark/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..678a9cdd068304c7124e990ce6e2398522d44a3b --- /dev/null +++ b/benchmark/evaluate.py @@ -0,0 +1,191 @@ +"""Evaluate an SynthScope detector on a benchmark track and write a schema-valid submission JSON. + + python benchmark/evaluate.py --model fat_former --track forensynths \ + --data /path/to/forensynths_test --out benchmark/submissions/fat_former.json + +Reuses the verified ``synthscope`` inference path (model build + checkpoint load + per-model +preprocessing + fake-probability), then computes per-generator + mean metrics with torchmetrics — +the same metrics the trainer reports. Image tracks only in v1 (video harness is future work; the +video seed entries come from their reproduction eval). + +Expected data layout (SynthScope common layout): + + //real/*.png //fake/*.png + +Use ``--reals DIR`` for benchmarks that share one real set across per-generator fake folders. +""" +from __future__ import annotations + +import argparse +import datetime +import json +from pathlib import Path + +import torch +from PIL import Image, ImageFile + +import synthscope + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"} +REAL_NAMES = {"real", "0_real", "reals"} +FAKE_NAMES = {"fake", "1_fake", "fakes", "ai"} + + +def _images(d: Path) -> list[Path]: + return sorted(p for p in d.rglob("*") if p.suffix.lower() in IMAGE_EXTS) if d and d.is_dir() else [] + + +def _find_split(gen_dir: Path, names: set[str]) -> Path | None: + for n in names: + if (gen_dir / n).is_dir(): + return gen_dir / n + return None + + +def discover_generators(root: Path, shared_reals: Path | None): + """Return [(generator_name, real_paths, fake_paths)] from the benchmark root.""" + shared = _images(shared_reals) if shared_reals else [] + out = [] + # Single pooled set: /real, /fake + if _find_split(root, REAL_NAMES) or _find_split(root, FAKE_NAMES): + reals = _images(_find_split(root, REAL_NAMES)) or shared + fakes = _images(_find_split(root, FAKE_NAMES)) + if reals and fakes: + return [("all", reals, fakes)] + for gen_dir in sorted(p for p in root.iterdir() if p.is_dir()): + reals = _images(_find_split(gen_dir, REAL_NAMES)) or shared + fakes = _images(_find_split(gen_dir, FAKE_NAMES)) + if reals and fakes: + out.append((gen_dir.name, reals, fakes)) + else: + print(f" skip generator '{gen_dir.name}': need both real and fake images") + return out + + +@torch.no_grad() +def score(det: "synthscope.Detector", paths: list[Path], batch_size: int) -> torch.Tensor: + """Return P(fake) for a list of image paths, using the detector's own preprocessing.""" + det._ensure_weights() + model = det._engine._get_model(det.spec) # build + load (verified path) + spec, device = det.spec, det._engine.device + probs = [] + for i in range(0, len(paths), batch_size): + batch = [] + for p in paths[i:i + batch_size]: + batch.append(spec.transform(Image.open(p).convert("RGB"))) + x = torch.stack(batch).to(device) + probs.append(det._engine._fake_prob(model, spec, x).detach().float().cpu()) + return torch.cat(probs) if probs else torch.empty(0) + + +def _per_generator_metrics(fake_probs: torch.Tensor, labels: torch.Tensor) -> dict | None: + from torchmetrics.functional import auroc, average_precision + + if labels.unique().numel() < 2: + return None # AUROC/AP need both classes + ap = float(average_precision(fake_probs, labels, task="binary")) * 100 + au = float(auroc(fake_probs, labels, task="binary")) * 100 + preds = (fake_probs >= 0.5).int() + real_acc = float((preds[labels == 0] == 0).float().mean()) * 100 + fake_acc = float((preds[labels == 1] == 1).float().mean()) * 100 + return {"ap": round(ap, 2), "auroc": round(au, 2), + "real_acc": round(real_acc, 2), "fake_acc": round(fake_acc, 2), + "balanced_acc": round((real_acc + fake_acc) / 2, 2)} + + +def _infer_eval_type(model_type: str) -> str: + from interface.backend.registry import get_spec + + spec = get_spec(model_type) + if str(spec.ckpt_kind).startswith("official"): + return "official-weights" + if spec.ckpt_kind == "none": + return "training-free" + return "from-scratch" + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Evaluate a detector and emit a benchmark submission.") + ap.add_argument("-m", "--model", required=True, help="SynthScope detector key (see `synthscope list`)") + ap.add_argument("--track", required=True, choices=["forensynths", "genimage-sdv1.4"], + help="benchmark track (image tracks only in v1)") + ap.add_argument("--data", required=True, help="benchmark root (per-generator real/fake dirs)") + ap.add_argument("--reals", default=None, help="shared real-image dir (for shared-real layouts)") + ap.add_argument("--out", required=True, help="output submission JSON path") + ap.add_argument("--batch-size", type=int, default=32) + ap.add_argument("--limit", type=int, default=None, help="cap images per class per generator") + ap.add_argument("--device", default=None) + ap.add_argument("--method", default=None, help="display name (default: registry name)") + ap.add_argument("--eval-type", default=None, + help="override provenance (official-weights/from-scratch/frozen-probe/...)") + ap.add_argument("--submitted-by", default="SynthScope") + args = ap.parse_args(argv) + + det = synthscope.load(args.model, device=args.device) + root = Path(args.data).expanduser() + gens = discover_generators(root, Path(args.reals).expanduser() if args.reals else None) + if not gens: + print(f"No generators with real+fake images under {root}") + return 1 + + per_gen, means = {}, {"ap": [], "auroc": [], "balanced_acc": []} + for name, reals, fakes in gens: + if args.limit: + reals, fakes = reals[:args.limit], fakes[:args.limit] + paths = reals + fakes + labels = torch.tensor([0] * len(reals) + [1] * len(fakes)) + print(f" {name}: {len(reals)} real + {len(fakes)} fake ...", flush=True) + probs = score(det, paths, args.batch_size) + m = _per_generator_metrics(probs, labels) + if m is None: + print(f" (skipped metrics: only one class)") + continue + per_gen[name] = m + for k in means: + means[k].append(m[k]) + + if not per_gen: + print("No generator produced valid metrics.") + return 1 + + def mean(xs): + return round(sum(xs) / len(xs), 2) + + from interface.backend.registry import get_spec + spec = get_spec(args.model) + submission = { + "method": args.method or spec.name, + "track": args.track, + "modality": "image", + "model_type": args.model, + "paper": spec.paper, + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": args.eval_type or _infer_eval_type(args.model), + "metrics": { + "acc": mean(means["balanced_acc"]), + "map": mean(means["ap"]), + "ap": None, + "auroc": mean(means["auroc"]), + }, + "per_generator": per_gen, + "submitted_by": args.submitted_by, + "date": datetime.date.today().isoformat(), + "verified": True, + "source": f"synthscope evaluate.py on {root} ({len(per_gen)} generators)", + "protocol_notes": None, + "synthscope_version": synthscope.__version__, + } + out = Path(args.out).expanduser() + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(submission, indent=2) + "\n") + print(f"Wrote {out}") + print(f" mAP={submission['metrics']['map']} AUROC={submission['metrics']['auroc']} " + f"Acc={submission['metrics']['acc']} over {len(per_gen)} generators") + print("Run `python benchmark/leaderboard.py` to refresh LEADERBOARD.md.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/generator_bench.py b/benchmark/generator_bench.py new file mode 100644 index 0000000000000000000000000000000000000000..62e5bf7a48768e5bfc90ae60115e71a7c3d0be99 --- /dev/null +++ b/benchmark/generator_bench.py @@ -0,0 +1,208 @@ +"""Generator benchmark: how well do SynthScope's image detectors catch images from ONE image generator? + +For every real image a text prompt is written (BLIP caption + a photographic style suffix), the generator creates a +new image from that prompt ALONE, and every available image detector scores the real and the generated images. The +result is a ranking by AUROC, shown in the web page's "Generator benchmarks" tab. + + # same 100 real images + prompts as the reference run, so generators are compared on identical content + python benchmark/generator_bench.py --generator flux2_klein --label "FLUX.2 [klein] 4B" \\ + --reuse benchmark/generator_runs/reference/manifest.json --out benchmark/generator_runs/flux2_klein_4b + # (the reference real images are not in git: add --real-root to find them by name) + + # build a fresh image set from your own real images + python benchmark/generator_bench.py --generator nanobanana --real-glob "/data/real/*.jpg" --n 100 \\ + --out benchmark/generator_runs/nanobanana + +``--generator`` is any key of ``interface/backend/generators.py`` that can create an image from text. Local open-weight +generators (FLUX.2 [klein], Qwen-Image-2.1) run in batch with one model load; hosted ones need their API key (``.env``). +Outputs in ``--out``: manifest.json, results.json, RESULTS.md (committed) and real/ + fake/ images (git-ignored). +Re-running skips images that already exist, so an interrupted run can be resumed. See docs/BENCHMARKS.md. +""" +import argparse +import glob +import json +import os +import random +import shutil +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +import numpy as np +from PIL import Image + +from interface.backend import generators as G +from interface.backend.registry import list_specs + +IMG_EXT = (".jpg", ".jpeg", ".png", ".webp") + + +def log(*a): + print(*a, flush=True) + + +def build_pairs(args, real_dir: Path) -> list: + """[{idx, caption, prompt, real_src}] and the real images copied to ``real_dir`` as 000.png, 001.png, ...""" + pairs = [] + if args.reuse: + src = json.load(open(args.reuse)) + src_real = Path(args.reuse).parent / "real" + found = None # file name -> path, built only if --real-root is needed + for p in src["pairs"][: args.n]: + png = src_real / f"{p['idx']:03d}.png" + dst = real_dir / f"{p['idx']:03d}.png" + if png.is_file(): + shutil.copyfile(png, dst) + elif args.real_root: # the real images are not in git: find them by file name + if found is None: + found = {f.name: f for root in args.real_root for f in Path(root).rglob("*") + if f.suffix.lower() in IMG_EXT} + match = found.get(Path(p.get("real_src", "")).name) + if match is None: + log(f" [{p['idx']}] real image {Path(p.get('real_src', '')).name} not found under --real-root, skipped") + continue + Image.open(match).convert("RGB").save(dst) + else: + continue + pairs.append({"idx": p["idx"], "caption": p.get("caption", ""), "prompt": p["prompt"], + "real_src": p.get("real_src", "")}) + log(f"[pairs] reusing {len(pairs)} prompts + real images from {args.reuse}") + return pairs + if not args.real_glob: + sys.exit("give --reuse MANIFEST or --real-glob PATTERN (real images to caption)") + cands = sorted({f for pat in args.real_glob for f in glob.glob(pat) if f.lower().endswith(IMG_EXT)}) + random.Random(args.seed).shuffle(cands) + log(f"[pairs] {len(cands)} candidate real images; need {args.n} (people are skipped)") + for src in cands: + if len(pairs) >= args.n: + break + data = open(src, "rb").read() + cap = G._caption(data) + if not cap or G._PERSON_RE.search(cap): # no photos of people in the benchmark + continue + i = len(pairs) + Image.open(src).convert("RGB").save(real_dir / f"{i:03d}.png") + pairs.append({"idx": i, "caption": cap, "prompt": cap + G.STYLE_SUFFIX, "real_src": src}) + return pairs + + +def generate(args, pairs: list, real_dir: Path, fake_dir: Path) -> list: + g = G.GENERATORS[args.generator] + if "recreate" not in g.modes: + sys.exit(f"{g.name} cannot create an image from text (modes: {g.modes})") + if not g.available(): + sys.exit(f"{g.name} is not available (API key / local setup missing, see docs/BENCHMARKS.md)") + model_key = getattr(g.call, "model_key", None) + t0 = time.time() + if model_key: # local open-weight model: one batch, one model load + items = [{"prompt": p["prompt"], "out": str(fake_dir / f"{p['idx']:03d}.png"), "seed": p["idx"], + "size": f"{args.res}x{args.res}"} for p in pairs] + G.local_batch(model_key, items) + else: # hosted generator: one API call per image + for p in pairs: + out = fake_dir / f"{p['idx']:03d}.png" + if out.exists(): + continue + try: + png = G.generate(args.generator, open(real_dir / f"{p['idx']:03d}.png", "rb").read(), p["prompt"], "recreate") + out.write_bytes(png) + log(f" [{p['idx']}] {p['caption']!r}") + except Exception as e: # blocked by a safety filter, rate limit, ... + log(f" [{p['idx']}] skipped: {type(e).__name__}: {e}") + kept = [p for p in pairs if (fake_dir / f"{p['idx']:03d}.png").is_file()] + for p in pairs: # keep real and fake balanced + if p not in kept and (real_dir / f"{p['idx']:03d}.png").exists(): + (real_dir / f"{p['idx']:03d}.png").unlink() + log(f"[generate] {len(kept)}/{len(pairs)} fakes ready ({time.time() - t0:.0f}s)") + return kept + + +def score(args, real_dir: Path, fake_dir: Path, out: Path) -> list: + from sklearn.metrics import average_precision_score, roc_auc_score + from interface.backend.engine import get_engine + engine = get_engine() + reals = [Image.open(p).convert("RGB") for p in sorted(glob.glob(str(real_dir / "*.png")))] + fakes = [Image.open(p).convert("RGB") for p in sorted(glob.glob(str(fake_dir / "*.png")))] + log(f"[score] {len(reals)} real + {len(fakes)} fake") + y = np.array([0] * len(reals) + [1] * len(fakes)) + keys = args.detectors.split(",") if args.detectors else None + rows = [] + for spec in list_specs(): + if spec.modality != "image" or (keys and spec.key not in keys): + continue + if not engine.is_available(spec): + log(f" {spec.key}: no checkpoint here, skipped") + continue + try: + t0 = time.time() + probs = np.array([engine.predict_image(spec, im)["fake_probability"] for im in reals + fakes], dtype="float64") + rows.append({ + "detector": spec.key, "name": spec.name, + "auroc": round(float(roc_auc_score(y, probs)) * 100, 2), + "ap": round(float(average_precision_score(y, probs)) * 100, 2), + "acc": round(float(((probs >= 0.5).astype(int) == y).mean()) * 100, 2), + "fake_detect_rate": round(float((probs[len(reals):] >= 0.5).mean()) * 100, 2), + "real_false_pos": round(float((probs[: len(reals)] >= 0.5).mean()) * 100, 2), + "mean_p_fake_on_fakes": round(float(probs[len(reals):].mean()), 4), + "mean_p_fake_on_reals": round(float(probs[: len(reals)].mean()), 4)}) + r = rows[-1] + log(f" {spec.key:12} AUROC={r['auroc']:6.2f} AP={r['ap']:6.2f} acc={r['acc']:6.2f} ({time.time() - t0:.0f}s)") + except Exception as e: + log(f" {spec.key}: ERROR {type(e).__name__}: {e}") + rows.sort(key=lambda r: r["auroc"], reverse=True) + json.dump(rows, open(out / "results.json", "w"), indent=2) + return rows + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--generator", required=True, help="key from interface/backend/generators.py, e.g. flux2_klein") + ap.add_argument("--label", help="name shown in the web page (default: the generator's name)") + ap.add_argument("--out", required=True, help="output directory, e.g. benchmark/generator_runs/flux2_klein_4b") + ap.add_argument("--reuse", help="manifest.json of an earlier run: reuse its real images + prompts (same content for every generator)") + ap.add_argument("--real-root", action="append", help="with --reuse: folder(s) to search for the reference run's real images by file " + "name (e.g. your GenImage and CNNDetection downloads), because the images themselves are not in the repo") + ap.add_argument("--real-glob", action="append", help="glob of real images to caption (repeatable); needed without --reuse") + ap.add_argument("--n", type=int, default=100, help="number of real/fake pairs") + ap.add_argument("--res", type=int, default=1024, help="output side in pixels for local generators") + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--detectors", help="comma-separated detector keys (default: every available image detector)") + ap.add_argument("--skip-generate", action="store_true", help="only (re)score the images already in --out") + args = ap.parse_args() + + out = Path(args.out).resolve() + real_dir, fake_dir = out / "real", out / "fake" + real_dir.mkdir(parents=True, exist_ok=True); fake_dir.mkdir(parents=True, exist_ok=True) + g = G.GENERATORS[args.generator] + manifest_path = out / "manifest.json" + if manifest_path.exists(): # resume + pairs = json.load(open(manifest_path))["pairs"] + for p in pairs: # real images may have been cleaned up after a partial run + src_png = Path(args.reuse).parent / "real" / f"{p['idx']:03d}.png" if args.reuse else None + if not (real_dir / f"{p['idx']:03d}.png").exists() and src_png and src_png.exists(): + shutil.copyfile(src_png, real_dir / f"{p['idx']:03d}.png") + else: + pairs = build_pairs(args, real_dir) + if not args.skip_generate: + pairs = generate(args, pairs, real_dir, fake_dir) + rows = score(args, real_dir, fake_dir, out) + label = args.label or g.name + manifest = {"label": label, "generator": args.generator, "n": len(pairs), "res": args.res, + "settings": f"{args.res}px · text-to-image from a caption", "created": time.strftime("%Y-%m-%d"), + "reuse_from": args.reuse or "", "pairs": pairs} + json.dump(manifest, open(manifest_path, "w"), indent=2) + md = [f"# Detectors vs {label}", "", + f"{len(pairs)} real images (people excluded) + {len(pairs)} images generated by {label} from a BLIP caption of each " + f"real image ({args.res}px). Ranked by AUROC; accuracy uses the 0.5 threshold.", "", + "| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % |", "|---|---|---|---|---|---|---|"] + md += [f"| {i} | {r['name']} | {r['auroc']} | {r['ap']} | {r['acc']} | {r['fake_detect_rate']} | {r['real_false_pos']} |" + for i, r in enumerate(rows, 1)] + (out / "RESULTS.md").write_text("\n".join(md) + "\n") + log("\n".join(md)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/generator_runs/_test_qwen/RESULTS.md b/benchmark/generator_runs/_test_qwen/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..df426374d92012793f3b6b7ff7edfdc30898502c --- /dev/null +++ b/benchmark/generator_runs/_test_qwen/RESULTS.md @@ -0,0 +1,19 @@ +# Detectors vs Qwen-Image-2.1 + +3 real images (people excluded) + 3 images generated by Qwen-Image-2.1 from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold. + +| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % | +|---|---|---|---|---|---|---| +| 1 | SAFE | 100.0 | 100.0 | 83.33 | 66.67 | 0.0 | +| 2 | AIDE | 100.0 | 100.0 | 83.33 | 66.67 | 0.0 | +| 3 | SPAI | 100.0 | 100.0 | 100.0 | 100.0 | 0.0 | +| 4 | VIB-Net | 100.0 | 100.0 | 100.0 | 100.0 | 0.0 | +| 5 | CO-SPY | 100.0 | 100.0 | 83.33 | 66.67 | 0.0 | +| 6 | PGC | 100.0 | 100.0 | 66.67 | 33.33 | 0.0 | +| 7 | CuRe | 100.0 | 100.0 | 66.67 | 33.33 | 0.0 | +| 8 | NPR | 83.33 | 75.0 | 66.67 | 100.0 | 66.67 | +| 9 | FatFormer | 77.78 | 86.67 | 83.33 | 66.67 | 0.0 | +| 10 | CNNDetection (ResNet-50) | 72.22 | 83.33 | 50.0 | 0.0 | 0.0 | +| 11 | Adapter-tuned DINOv2 | 72.22 | 83.33 | 50.0 | 0.0 | 0.0 | +| 12 | UnivFD / CLIP | 44.44 | 55.56 | 50.0 | 0.0 | 0.0 | +| 13 | DINOv2 probe | 44.44 | 53.33 | 50.0 | 0.0 | 0.0 | diff --git a/benchmark/generator_runs/_test_qwen/manifest.json b/benchmark/generator_runs/_test_qwen/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..757fc72357960c059d68b645735ebbfe8bd6a72b --- /dev/null +++ b/benchmark/generator_runs/_test_qwen/manifest.json @@ -0,0 +1,29 @@ +{ + "label": "Qwen-Image-2.1", + "generator": "qwen_image_2_1", + "n": 3, + "res": 1024, + "settings": "1024px \u00b7 text-to-image from a caption", + "created": "2026-10-04", + "reuse_from": "benchmark/generator_runs/reference/manifest.json", + "pairs": [ + { + "idx": 0, + "caption": "sheep is white", + "prompt": "sheep is white, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG" + }, + { + "idx": 1, + "caption": "a bird sitting on a branch in a tree", + "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG" + }, + { + "idx": 2, + "caption": "a green grasshopper on a leaf", + "prompt": "a green grasshopper on a leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG" + } + ] +} \ No newline at end of file diff --git a/benchmark/generator_runs/_test_qwen/results.json b/benchmark/generator_runs/_test_qwen/results.json new file mode 100644 index 0000000000000000000000000000000000000000..1128d284d9596a9116c0c82a0e27b05d6f6069cb --- /dev/null +++ b/benchmark/generator_runs/_test_qwen/results.json @@ -0,0 +1,145 @@ +[ + { + "detector": "safe", + "name": "SAFE", + "auroc": 100.0, + "ap": 100.0, + "acc": 83.33, + "fake_detect_rate": 66.67, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.5512, + "mean_p_fake_on_reals": 0.0758 + }, + { + "detector": "aide", + "name": "AIDE", + "auroc": 100.0, + "ap": 100.0, + "acc": 83.33, + "fake_detect_rate": 66.67, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.623, + "mean_p_fake_on_reals": 0.0486 + }, + { + "detector": "spai", + "name": "SPAI", + "auroc": 100.0, + "ap": 100.0, + "acc": 100.0, + "fake_detect_rate": 100.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.984, + "mean_p_fake_on_reals": 0.0026 + }, + { + "detector": "vib_net", + "name": "VIB-Net", + "auroc": 100.0, + "ap": 100.0, + "acc": 100.0, + "fake_detect_rate": 100.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.9034, + "mean_p_fake_on_reals": 0.0684 + }, + { + "detector": "co_spy", + "name": "CO-SPY", + "auroc": 100.0, + "ap": 100.0, + "acc": 83.33, + "fake_detect_rate": 66.67, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.7782, + "mean_p_fake_on_reals": 0.0484 + }, + { + "detector": "pgc", + "name": "PGC", + "auroc": 100.0, + "ap": 100.0, + "acc": 66.67, + "fake_detect_rate": 33.33, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.4674, + "mean_p_fake_on_reals": 0.2349 + }, + { + "detector": "cure", + "name": "CuRe", + "auroc": 100.0, + "ap": 100.0, + "acc": 66.67, + "fake_detect_rate": 33.33, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.5297, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "npr", + "name": "NPR", + "auroc": 83.33, + "ap": 75.0, + "acc": 66.67, + "fake_detect_rate": 100.0, + "real_false_pos": 66.67, + "mean_p_fake_on_fakes": 1.0, + "mean_p_fake_on_reals": 0.6673 + }, + { + "detector": "fat_former", + "name": "FatFormer", + "auroc": 77.78, + "ap": 86.67, + "acc": 83.33, + "fake_detect_rate": 66.67, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.6629, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "cnn", + "name": "CNNDetection (ResNet-50)", + "auroc": 72.22, + "ap": 83.33, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.0003, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "adapted_dino", + "name": "Adapter-tuned DINOv2", + "auroc": 72.22, + "ap": 83.33, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.0, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "clip", + "name": "UnivFD / CLIP", + "auroc": 44.44, + "ap": 55.56, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.1024, + "mean_p_fake_on_reals": 0.1369 + }, + { + "detector": "dinov2", + "name": "DINOv2 probe", + "auroc": 44.44, + "ap": 53.33, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.0132, + "mean_p_fake_on_reals": 0.0194 + } +] \ No newline at end of file diff --git a/benchmark/generator_runs/flux2_klein_4b/RESULTS.md b/benchmark/generator_runs/flux2_klein_4b/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..a3f101bba5de2516a47faa4bc48a80b545fb8c96 --- /dev/null +++ b/benchmark/generator_runs/flux2_klein_4b/RESULTS.md @@ -0,0 +1,20 @@ +# Detectors vs FLUX.2 [klein] 4B + +100 real images (people excluded) + 100 images generated by FLUX.2 [klein] 4B from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold. + +| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % | +|---|---|---|---|---|---|---| +| 1 | VIB-Net | 99.91 | 99.91 | 92.0 | 100.0 | 16.0 | +| 2 | CO-SPY | 99.31 | 99.27 | 98.0 | 99.0 | 3.0 | +| 3 | CuRe | 98.6 | 98.91 | 92.5 | 85.0 | 0.0 | +| 4 | PGC | 97.33 | 96.57 | 53.5 | 7.0 | 0.0 | +| 5 | AIDE | 94.08 | 90.5 | 81.0 | 69.0 | 7.0 | +| 6 | SAFE | 92.04 | 82.32 | 87.0 | 84.0 | 10.0 | +| 7 | NPR | 89.18 | 82.53 | 78.0 | 99.0 | 43.0 | +| 8 | SPAI | 78.08 | 75.23 | 65.0 | 38.0 | 8.0 | +| 9 | DINOv3 probe | 51.06 | 49.11 | 47.5 | 8.0 | 13.0 | +| 10 | FatFormer | 46.01 | 48.06 | 50.5 | 5.0 | 4.0 | +| 11 | Adapter-tuned DINOv2 | 42.62 | 47.55 | 50.0 | 0.0 | 0.0 | +| 12 | CNNDetection (ResNet-50) | 40.84 | 45.42 | 50.5 | 2.0 | 1.0 | +| 13 | DINOv2 probe | 38.52 | 43.2 | 47.0 | 7.0 | 13.0 | +| 14 | UnivFD / CLIP | 26.8 | 36.87 | 47.0 | 1.0 | 7.0 | diff --git a/benchmark/generator_runs/flux2_klein_4b/manifest.json b/benchmark/generator_runs/flux2_klein_4b/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ae1e21fed9d0cb71d25584bb236420521b3b83f3 --- /dev/null +++ b/benchmark/generator_runs/flux2_klein_4b/manifest.json @@ -0,0 +1,611 @@ +{ + "label": "FLUX.2 [klein] 4B", + "generator": "flux2_klein", + "n": 100, + "res": 1024, + "settings": "1024px \u00b7 text-to-image from a caption", + "created": "2026-10-04", + "reuse_from": "benchmark/generator_runs/reference/manifest.json", + "pairs": [ + { + "idx": 0, + "caption": "sheep is white", + "prompt": "sheep is white, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG" + }, + { + "idx": 1, + "caption": "a bird sitting on a branch in a tree", + "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG" + }, + { + "idx": 2, + "caption": "a green grasshopper on a leaf", + "prompt": "a green grasshopper on a leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG" + }, + { + "idx": 3, + "caption": "the bell is black", + "prompt": "the bell is black, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG" + }, + { + "idx": 4, + "caption": "a car driving down a city street in the middle of the day", + "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png" + }, + { + "idx": 5, + "caption": "a dog is standing in the grass", + "prompt": "a dog is standing in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG" + }, + { + "idx": 6, + "caption": "a small kitten walking across a rocky beach", + "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG" + }, + { + "idx": 7, + "caption": "a mountain covered in snow", + "prompt": "a mountain covered in snow, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG" + }, + { + "idx": 8, + "caption": "a sailer sailing in the ocean near a lighthouse", + "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG" + }, + { + "idx": 9, + "caption": "a bowl of masoor", + "prompt": "a bowl of masoor, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG" + }, + { + "idx": 10, + "caption": "a green bird house", + "prompt": "a green bird house, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG" + }, + { + "idx": 11, + "caption": "a body of water", + "prompt": "a body of water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG" + }, + { + "idx": 12, + "caption": "a dog in a pool", + "prompt": "a dog in a pool, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG" + }, + { + "idx": 13, + "caption": "two tools, one with a wren and the other with a wren", + "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG" + }, + { + "idx": 14, + "caption": "a black leather folder with a red trim", + "prompt": "a black leather folder with a red trim, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png" + }, + { + "idx": 15, + "caption": "a small bird sitting on a wooden surface", + "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG" + }, + { + "idx": 16, + "caption": "mouse mouse on mouse mouse mouse", + "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG" + }, + { + "idx": 17, + "caption": "a small metal object on a wooden table", + "prompt": "a small metal object on a wooden table, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG" + }, + { + "idx": 18, + "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour", + "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG" + }, + { + "idx": 19, + "caption": "a white rabbit in a cage", + "prompt": "a white rabbit in a cage, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG" + }, + { + "idx": 20, + "caption": "a grass bug on a green leaf", + "prompt": "a grass bug on a green leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG" + }, + { + "idx": 21, + "caption": "a field of grass", + "prompt": "a field of grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG" + }, + { + "idx": 22, + "caption": "a black padlock with a key", + "prompt": "a black padlock with a key, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png" + }, + { + "idx": 23, + "caption": "a group of balls", + "prompt": "a group of balls, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG" + }, + { + "idx": 24, + "caption": "three dogs standing on a hill", + "prompt": "three dogs standing on a hill, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG" + }, + { + "idx": 25, + "caption": "a dog standing in a field", + "prompt": "a dog standing in a field, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG" + }, + { + "idx": 26, + "caption": "a pink toilet", + "prompt": "a pink toilet, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG" + }, + { + "idx": 27, + "caption": "the sky is cloudy", + "prompt": "the sky is cloudy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG" + }, + { + "idx": 28, + "caption": "a bird is standing on the ground", + "prompt": "a bird is standing on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG" + }, + { + "idx": 29, + "caption": "a leopard sitting on a tree branch", + "prompt": "a leopard sitting on a tree branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG" + }, + { + "idx": 30, + "caption": "an old barn with a broken roof", + "prompt": "an old barn with a broken roof, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG" + }, + { + "idx": 31, + "caption": "a slug crawling on a concrete surface", + "prompt": "a slug crawling on a concrete surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG" + }, + { + "idx": 32, + "caption": "the three bottles of the serums are shown in the image", + "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png" + }, + { + "idx": 33, + "caption": "a bear cub", + "prompt": "a bear cub, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG" + }, + { + "idx": 34, + "caption": "a street with a yellow line painted on the road", + "prompt": "a street with a yellow line painted on the road, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png" + }, + { + "idx": 35, + "caption": "a lion fish swimming in the water", + "prompt": "a lion fish swimming in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG" + }, + { + "idx": 36, + "caption": "a plastic container with food", + "prompt": "a plastic container with food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png" + }, + { + "idx": 37, + "caption": "a car is driving down a street in a city", + "prompt": "a car is driving down a street in a city, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png" + }, + { + "idx": 38, + "caption": "a wooden desk", + "prompt": "a wooden desk, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG" + }, + { + "idx": 39, + "caption": "a bird standing in the water", + "prompt": "a bird standing in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG" + }, + { + "idx": 40, + "caption": "two parking meters", + "prompt": "two parking meters, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG" + }, + { + "idx": 41, + "caption": "a butterfly on a yellow flower", + "prompt": "a butterfly on a yellow flower, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG" + }, + { + "idx": 42, + "caption": "a green snake on a rock", + "prompt": "a green snake on a rock, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG" + }, + { + "idx": 43, + "caption": "a green snake in a tree", + "prompt": "a green snake in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG" + }, + { + "idx": 44, + "caption": "purple and gold beaded bracelet", + "prompt": "purple and gold beaded bracelet, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG" + }, + { + "idx": 45, + "caption": "a dog standing in the shade", + "prompt": "a dog standing in the shade, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG" + }, + { + "idx": 46, + "caption": "a cliff with a cliff face", + "prompt": "a cliff with a cliff face, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG" + }, + { + "idx": 47, + "caption": "a dog laying in the grass", + "prompt": "a dog laying in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG" + }, + { + "idx": 48, + "caption": "a small dog sitting on a bench", + "prompt": "a small dog sitting on a bench, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG" + }, + { + "idx": 49, + "caption": "a dog standing on a hill", + "prompt": "a dog standing on a hill, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG" + }, + { + "idx": 50, + "caption": "a white plate with food on it", + "prompt": "a white plate with food on it, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG" + }, + { + "idx": 51, + "caption": "a wooden chair", + "prompt": "a wooden chair, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG" + }, + { + "idx": 52, + "caption": "a large church with a stained glass window", + "prompt": "a large church with a stained glass window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG" + }, + { + "idx": 53, + "caption": "a dog running in the grass", + "prompt": "a dog running in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG" + }, + { + "idx": 54, + "caption": "a hand holding a hot dog with lete and lete", + "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png" + }, + { + "idx": 55, + "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish", + "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG" + }, + { + "idx": 56, + "caption": "a brown dog with a long tongue", + "prompt": "a brown dog with a long tongue, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG" + }, + { + "idx": 57, + "caption": "a wooden bench", + "prompt": "a wooden bench, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG" + }, + { + "idx": 58, + "caption": "a white dog toy", + "prompt": "a white dog toy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG" + }, + { + "idx": 59, + "caption": "a lake with trees and water in the background", + "prompt": "a lake with trees and water in the background, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG" + }, + { + "idx": 60, + "caption": "a crab crawling on the ground", + "prompt": "a crab crawling on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG" + }, + { + "idx": 61, + "caption": "a city street with a lot of buildings", + "prompt": "a city street with a lot of buildings, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png" + }, + { + "idx": 62, + "caption": "old wooden barrels stock photo, old wooden barrels by andy", + "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG" + }, + { + "idx": 63, + "caption": "a green beetle with red and green stripes", + "prompt": "a green beetle with red and green stripes, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG" + }, + { + "idx": 64, + "caption": "a plate of food", + "prompt": "a plate of food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG" + }, + { + "idx": 65, + "caption": "a fish in a tank", + "prompt": "a fish in a tank, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG" + }, + { + "idx": 66, + "caption": "a street with a lot of luggage and palm trees", + "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png" + }, + { + "idx": 67, + "caption": "a tree in the middle of a city", + "prompt": "a tree in the middle of a city, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png" + }, + { + "idx": 68, + "caption": "a kitchen with a large white island and a large window", + "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png" + }, + { + "idx": 69, + "caption": "a small brown and black dog standing on a white surface", + "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG" + }, + { + "idx": 70, + "caption": "a train on the tracks", + "prompt": "a train on the tracks, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG" + }, + { + "idx": 71, + "caption": "a large circular window", + "prompt": "a large circular window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG" + }, + { + "idx": 72, + "caption": "a machine in a factory", + "prompt": "a machine in a factory, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG" + }, + { + "idx": 73, + "caption": "a red boat docked in a harbor", + "prompt": "a red boat docked in a harbor, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG" + }, + { + "idx": 74, + "caption": "a bird perched on a branch", + "prompt": "a bird perched on a branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG" + }, + { + "idx": 75, + "caption": "a large gray elephant", + "prompt": "a large gray elephant, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG" + }, + { + "idx": 76, + "caption": "a street with a star painted on the sidewalk", + "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png" + }, + { + "idx": 77, + "caption": "a fish in a tank with a fish inside", + "prompt": "a fish in a tank with a fish inside, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG" + }, + { + "idx": 78, + "caption": "a pot of soup", + "prompt": "a pot of soup, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG" + }, + { + "idx": 79, + "caption": "a bunch of bubbles", + "prompt": "a bunch of bubbles, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG" + }, + { + "idx": 80, + "caption": "a vase on display", + "prompt": "a vase on display, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG" + }, + { + "idx": 81, + "caption": "a toilet with a toilet seat and a toilet roll", + "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG" + }, + { + "idx": 82, + "caption": "a piece of food", + "prompt": "a piece of food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG" + }, + { + "idx": 83, + "caption": "a car is parked in a street with a bus in the background", + "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png" + }, + { + "idx": 84, + "caption": "a white car driving down a street next to a building", + "prompt": "a white car driving down a street next to a building, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png" + }, + { + "idx": 85, + "caption": "a small canal in venice, italy", + "prompt": "a small canal in venice, italy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG" + }, + { + "idx": 86, + "caption": "a black velvet jacket", + "prompt": "a black velvet jacket, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG" + }, + { + "idx": 87, + "caption": "a bear is sitting in a tree branch", + "prompt": "a bear is sitting in a tree branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG" + }, + { + "idx": 88, + "caption": "a goat standing in the woods", + "prompt": "a goat standing in the woods, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG" + }, + { + "idx": 89, + "caption": "a military vehicle", + "prompt": "a military vehicle, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG" + }, + { + "idx": 90, + "caption": "a cup with a bunch of pencils in it", + "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG" + }, + { + "idx": 91, + "caption": "a jaguar in the water", + "prompt": "a jaguar in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG" + }, + { + "idx": 92, + "caption": "a flower made out of felt and felt", + "prompt": "a flower made out of felt and felt, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG" + }, + { + "idx": 93, + "caption": "a dog with a pink shirt on", + "prompt": "a dog with a pink shirt on, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG" + }, + { + "idx": 94, + "caption": "a close up of a bug on a rock", + "prompt": "a close up of a bug on a rock, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG" + }, + { + "idx": 95, + "caption": "a close up of a bird with a yellow eye", + "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG" + }, + { + "idx": 96, + "caption": "a hyen walking down a dirt road", + "prompt": "a hyen walking down a dirt road, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG" + }, + { + "idx": 97, + "caption": "a col of pictures of food including vegetables and vegetables", + "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG" + }, + { + "idx": 98, + "caption": "a brown barn", + "prompt": "a brown barn, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG" + }, + { + "idx": 99, + "caption": "a small black and white dog sitting on the ground", + "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG" + } + ] +} \ No newline at end of file diff --git a/benchmark/generator_runs/flux2_klein_4b/results.json b/benchmark/generator_runs/flux2_klein_4b/results.json new file mode 100644 index 0000000000000000000000000000000000000000..19746c5032d2ada4e03d6befe96462e317abf3be --- /dev/null +++ b/benchmark/generator_runs/flux2_klein_4b/results.json @@ -0,0 +1,156 @@ +[ + { + "detector": "vib_net", + "name": "VIB-Net", + "auroc": 99.91, + "ap": 99.91, + "acc": 92.0, + "fake_detect_rate": 100.0, + "real_false_pos": 16.0, + "mean_p_fake_on_fakes": 0.9752, + "mean_p_fake_on_reals": 0.2301 + }, + { + "detector": "co_spy", + "name": "CO-SPY", + "auroc": 99.31, + "ap": 99.27, + "acc": 98.0, + "fake_detect_rate": 99.0, + "real_false_pos": 3.0, + "mean_p_fake_on_fakes": 0.847, + "mean_p_fake_on_reals": 0.116 + }, + { + "detector": "cure", + "name": "CuRe", + "auroc": 98.6, + "ap": 98.91, + "acc": 92.5, + "fake_detect_rate": 85.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.8466, + "mean_p_fake_on_reals": 0.0129 + }, + { + "detector": "pgc", + "name": "PGC", + "auroc": 97.33, + "ap": 96.57, + "acc": 53.5, + "fake_detect_rate": 7.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.438, + "mean_p_fake_on_reals": 0.2672 + }, + { + "detector": "aide", + "name": "AIDE", + "auroc": 94.08, + "ap": 90.5, + "acc": 81.0, + "fake_detect_rate": 69.0, + "real_false_pos": 7.0, + "mean_p_fake_on_fakes": 0.6593, + "mean_p_fake_on_reals": 0.1102 + }, + { + "detector": "safe", + "name": "SAFE", + "auroc": 92.04, + "ap": 82.32, + "acc": 87.0, + "fake_detect_rate": 84.0, + "real_false_pos": 10.0, + "mean_p_fake_on_fakes": 0.7562, + "mean_p_fake_on_reals": 0.1425 + }, + { + "detector": "npr", + "name": "NPR", + "auroc": 89.18, + "ap": 82.53, + "acc": 78.0, + "fake_detect_rate": 99.0, + "real_false_pos": 43.0, + "mean_p_fake_on_fakes": 0.99, + "mean_p_fake_on_reals": 0.4365 + }, + { + "detector": "spai", + "name": "SPAI", + "auroc": 78.08, + "ap": 75.23, + "acc": 65.0, + "fake_detect_rate": 38.0, + "real_false_pos": 8.0, + "mean_p_fake_on_fakes": 0.368, + "mean_p_fake_on_reals": 0.0778 + }, + { + "detector": "dinov3", + "name": "DINOv3 probe", + "auroc": 51.06, + "ap": 49.11, + "acc": 47.5, + "fake_detect_rate": 8.0, + "real_false_pos": 13.0, + "mean_p_fake_on_fakes": 0.2746, + "mean_p_fake_on_reals": 0.2792 + }, + { + "detector": "fat_former", + "name": "FatFormer", + "auroc": 46.01, + "ap": 48.06, + "acc": 50.5, + "fake_detect_rate": 5.0, + "real_false_pos": 4.0, + "mean_p_fake_on_fakes": 0.0523, + "mean_p_fake_on_reals": 0.0412 + }, + { + "detector": "adapted_dino", + "name": "Adapter-tuned DINOv2", + "auroc": 42.62, + "ap": 47.55, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.0001, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "cnn", + "name": "CNNDetection (ResNet-50)", + "auroc": 40.84, + "ap": 45.42, + "acc": 50.5, + "fake_detect_rate": 2.0, + "real_false_pos": 1.0, + "mean_p_fake_on_fakes": 0.0169, + "mean_p_fake_on_reals": 0.0083 + }, + { + "detector": "dinov2", + "name": "DINOv2 probe", + "auroc": 38.52, + "ap": 43.2, + "acc": 47.0, + "fake_detect_rate": 7.0, + "real_false_pos": 13.0, + "mean_p_fake_on_fakes": 0.1156, + "mean_p_fake_on_reals": 0.1751 + }, + { + "detector": "clip", + "name": "UnivFD / CLIP", + "auroc": 26.8, + "ap": 36.87, + "acc": 47.0, + "fake_detect_rate": 1.0, + "real_false_pos": 7.0, + "mean_p_fake_on_fakes": 0.0728, + "mean_p_fake_on_reals": 0.1656 + } +] \ No newline at end of file diff --git a/benchmark/generator_runs/flux2_klein_9b/RESULTS.md b/benchmark/generator_runs/flux2_klein_9b/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..0c04ca5b716d67027106a9c813d2d0c852beae2f --- /dev/null +++ b/benchmark/generator_runs/flux2_klein_9b/RESULTS.md @@ -0,0 +1,20 @@ +# Detectors vs FLUX.2 [klein] 9B + +100 real images (people excluded) + 100 images generated by FLUX.2 [klein] 9B from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold. + +| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % | +|---|---|---|---|---|---|---| +| 1 | VIB-Net | 99.73 | 99.73 | 92.0 | 100.0 | 16.0 | +| 2 | CO-SPY | 99.34 | 99.3 | 98.0 | 99.0 | 3.0 | +| 3 | PGC | 97.35 | 96.45 | 53.5 | 7.0 | 0.0 | +| 4 | CuRe | 96.2 | 97.04 | 88.5 | 77.0 | 0.0 | +| 5 | AIDE | 93.73 | 88.94 | 81.0 | 69.0 | 7.0 | +| 6 | SAFE | 92.32 | 84.09 | 88.5 | 87.0 | 10.0 | +| 7 | NPR | 88.61 | 81.74 | 78.5 | 100.0 | 43.0 | +| 8 | SPAI | 66.54 | 63.78 | 56.0 | 20.0 | 8.0 | +| 9 | FatFormer | 45.86 | 46.65 | 50.5 | 5.0 | 4.0 | +| 10 | DINOv3 probe | 45.76 | 45.57 | 46.0 | 5.0 | 13.0 | +| 11 | Adapter-tuned DINOv2 | 44.13 | 46.55 | 50.0 | 0.0 | 0.0 | +| 12 | CNNDetection (ResNet-50) | 40.86 | 46.29 | 50.0 | 1.0 | 1.0 | +| 13 | DINOv2 probe | 35.29 | 41.64 | 46.0 | 5.0 | 13.0 | +| 14 | UnivFD / CLIP | 26.07 | 36.57 | 47.0 | 1.0 | 7.0 | diff --git a/benchmark/generator_runs/flux2_klein_9b/manifest.json b/benchmark/generator_runs/flux2_klein_9b/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3facef7d8c1620307879a14f73842f8f1c28de8d --- /dev/null +++ b/benchmark/generator_runs/flux2_klein_9b/manifest.json @@ -0,0 +1,611 @@ +{ + "label": "FLUX.2 [klein] 9B", + "generator": "flux2_klein_9b", + "n": 100, + "res": 1024, + "settings": "1024px \u00b7 text-to-image from a caption", + "created": "2026-10-04", + "reuse_from": "benchmark/generator_runs/reference/manifest.json", + "pairs": [ + { + "idx": 0, + "caption": "sheep is white", + "prompt": "sheep is white, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG" + }, + { + "idx": 1, + "caption": "a bird sitting on a branch in a tree", + "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG" + }, + { + "idx": 2, + "caption": "a green grasshopper on a leaf", + "prompt": "a green grasshopper on a leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG" + }, + { + "idx": 3, + "caption": "the bell is black", + "prompt": "the bell is black, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG" + }, + { + "idx": 4, + "caption": "a car driving down a city street in the middle of the day", + "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png" + }, + { + "idx": 5, + "caption": "a dog is standing in the grass", + "prompt": "a dog is standing in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG" + }, + { + "idx": 6, + "caption": "a small kitten walking across a rocky beach", + "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG" + }, + { + "idx": 7, + "caption": "a mountain covered in snow", + "prompt": "a mountain covered in snow, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG" + }, + { + "idx": 8, + "caption": "a sailer sailing in the ocean near a lighthouse", + "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG" + }, + { + "idx": 9, + "caption": "a bowl of masoor", + "prompt": "a bowl of masoor, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG" + }, + { + "idx": 10, + "caption": "a green bird house", + "prompt": "a green bird house, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG" + }, + { + "idx": 11, + "caption": "a body of water", + "prompt": "a body of water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG" + }, + { + "idx": 12, + "caption": "a dog in a pool", + "prompt": "a dog in a pool, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG" + }, + { + "idx": 13, + "caption": "two tools, one with a wren and the other with a wren", + "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG" + }, + { + "idx": 14, + "caption": "a black leather folder with a red trim", + "prompt": "a black leather folder with a red trim, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png" + }, + { + "idx": 15, + "caption": "a small bird sitting on a wooden surface", + "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG" + }, + { + "idx": 16, + "caption": "mouse mouse on mouse mouse mouse", + "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG" + }, + { + "idx": 17, + "caption": "a small metal object on a wooden table", + "prompt": "a small metal object on a wooden table, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG" + }, + { + "idx": 18, + "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour", + "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG" + }, + { + "idx": 19, + "caption": "a white rabbit in a cage", + "prompt": "a white rabbit in a cage, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG" + }, + { + "idx": 20, + "caption": "a grass bug on a green leaf", + "prompt": "a grass bug on a green leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG" + }, + { + "idx": 21, + "caption": "a field of grass", + "prompt": "a field of grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG" + }, + { + "idx": 22, + "caption": "a black padlock with a key", + "prompt": "a black padlock with a key, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png" + }, + { + "idx": 23, + "caption": "a group of balls", + "prompt": "a group of balls, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG" + }, + { + "idx": 24, + "caption": "three dogs standing on a hill", + "prompt": "three dogs standing on a hill, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG" + }, + { + "idx": 25, + "caption": "a dog standing in a field", + "prompt": "a dog standing in a field, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG" + }, + { + "idx": 26, + "caption": "a pink toilet", + "prompt": "a pink toilet, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG" + }, + { + "idx": 27, + "caption": "the sky is cloudy", + "prompt": "the sky is cloudy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG" + }, + { + "idx": 28, + "caption": "a bird is standing on the ground", + "prompt": "a bird is standing on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG" + }, + { + "idx": 29, + "caption": "a leopard sitting on a tree branch", + "prompt": "a leopard sitting on a tree branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG" + }, + { + "idx": 30, + "caption": "an old barn with a broken roof", + "prompt": "an old barn with a broken roof, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG" + }, + { + "idx": 31, + "caption": "a slug crawling on a concrete surface", + "prompt": "a slug crawling on a concrete surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG" + }, + { + "idx": 32, + "caption": "the three bottles of the serums are shown in the image", + "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png" + }, + { + "idx": 33, + "caption": "a bear cub", + "prompt": "a bear cub, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG" + }, + { + "idx": 34, + "caption": "a street with a yellow line painted on the road", + "prompt": "a street with a yellow line painted on the road, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png" + }, + { + "idx": 35, + "caption": "a lion fish swimming in the water", + "prompt": "a lion fish swimming in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG" + }, + { + "idx": 36, + "caption": "a plastic container with food", + "prompt": "a plastic container with food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png" + }, + { + "idx": 37, + "caption": "a car is driving down a street in a city", + "prompt": "a car is driving down a street in a city, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png" + }, + { + "idx": 38, + "caption": "a wooden desk", + "prompt": "a wooden desk, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG" + }, + { + "idx": 39, + "caption": "a bird standing in the water", + "prompt": "a bird standing in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG" + }, + { + "idx": 40, + "caption": "two parking meters", + "prompt": "two parking meters, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG" + }, + { + "idx": 41, + "caption": "a butterfly on a yellow flower", + "prompt": "a butterfly on a yellow flower, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG" + }, + { + "idx": 42, + "caption": "a green snake on a rock", + "prompt": "a green snake on a rock, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG" + }, + { + "idx": 43, + "caption": "a green snake in a tree", + "prompt": "a green snake in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG" + }, + { + "idx": 44, + "caption": "purple and gold beaded bracelet", + "prompt": "purple and gold beaded bracelet, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG" + }, + { + "idx": 45, + "caption": "a dog standing in the shade", + "prompt": "a dog standing in the shade, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG" + }, + { + "idx": 46, + "caption": "a cliff with a cliff face", + "prompt": "a cliff with a cliff face, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG" + }, + { + "idx": 47, + "caption": "a dog laying in the grass", + "prompt": "a dog laying in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG" + }, + { + "idx": 48, + "caption": "a small dog sitting on a bench", + "prompt": "a small dog sitting on a bench, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG" + }, + { + "idx": 49, + "caption": "a dog standing on a hill", + "prompt": "a dog standing on a hill, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG" + }, + { + "idx": 50, + "caption": "a white plate with food on it", + "prompt": "a white plate with food on it, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG" + }, + { + "idx": 51, + "caption": "a wooden chair", + "prompt": "a wooden chair, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG" + }, + { + "idx": 52, + "caption": "a large church with a stained glass window", + "prompt": "a large church with a stained glass window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG" + }, + { + "idx": 53, + "caption": "a dog running in the grass", + "prompt": "a dog running in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG" + }, + { + "idx": 54, + "caption": "a hand holding a hot dog with lete and lete", + "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png" + }, + { + "idx": 55, + "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish", + "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG" + }, + { + "idx": 56, + "caption": "a brown dog with a long tongue", + "prompt": "a brown dog with a long tongue, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG" + }, + { + "idx": 57, + "caption": "a wooden bench", + "prompt": "a wooden bench, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG" + }, + { + "idx": 58, + "caption": "a white dog toy", + "prompt": "a white dog toy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG" + }, + { + "idx": 59, + "caption": "a lake with trees and water in the background", + "prompt": "a lake with trees and water in the background, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG" + }, + { + "idx": 60, + "caption": "a crab crawling on the ground", + "prompt": "a crab crawling on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG" + }, + { + "idx": 61, + "caption": "a city street with a lot of buildings", + "prompt": "a city street with a lot of buildings, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png" + }, + { + "idx": 62, + "caption": "old wooden barrels stock photo, old wooden barrels by andy", + "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG" + }, + { + "idx": 63, + "caption": "a green beetle with red and green stripes", + "prompt": "a green beetle with red and green stripes, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG" + }, + { + "idx": 64, + "caption": "a plate of food", + "prompt": "a plate of food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG" + }, + { + "idx": 65, + "caption": "a fish in a tank", + "prompt": "a fish in a tank, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG" + }, + { + "idx": 66, + "caption": "a street with a lot of luggage and palm trees", + "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png" + }, + { + "idx": 67, + "caption": "a tree in the middle of a city", + "prompt": "a tree in the middle of a city, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png" + }, + { + "idx": 68, + "caption": "a kitchen with a large white island and a large window", + "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png" + }, + { + "idx": 69, + "caption": "a small brown and black dog standing on a white surface", + "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG" + }, + { + "idx": 70, + "caption": "a train on the tracks", + "prompt": "a train on the tracks, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG" + }, + { + "idx": 71, + "caption": "a large circular window", + "prompt": "a large circular window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG" + }, + { + "idx": 72, + "caption": "a machine in a factory", + "prompt": "a machine in a factory, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG" + }, + { + "idx": 73, + "caption": "a red boat docked in a harbor", + "prompt": "a red boat docked in a harbor, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG" + }, + { + "idx": 74, + "caption": "a bird perched on a branch", + "prompt": "a bird perched on a branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG" + }, + { + "idx": 75, + "caption": "a large gray elephant", + "prompt": "a large gray elephant, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG" + }, + { + "idx": 76, + "caption": "a street with a star painted on the sidewalk", + "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png" + }, + { + "idx": 77, + "caption": "a fish in a tank with a fish inside", + "prompt": "a fish in a tank with a fish inside, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG" + }, + { + "idx": 78, + "caption": "a pot of soup", + "prompt": "a pot of soup, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG" + }, + { + "idx": 79, + "caption": "a bunch of bubbles", + "prompt": "a bunch of bubbles, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG" + }, + { + "idx": 80, + "caption": "a vase on display", + "prompt": "a vase on display, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG" + }, + { + "idx": 81, + "caption": "a toilet with a toilet seat and a toilet roll", + "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG" + }, + { + "idx": 82, + "caption": "a piece of food", + "prompt": "a piece of food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG" + }, + { + "idx": 83, + "caption": "a car is parked in a street with a bus in the background", + "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png" + }, + { + "idx": 84, + "caption": "a white car driving down a street next to a building", + "prompt": "a white car driving down a street next to a building, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png" + }, + { + "idx": 85, + "caption": "a small canal in venice, italy", + "prompt": "a small canal in venice, italy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG" + }, + { + "idx": 86, + "caption": "a black velvet jacket", + "prompt": "a black velvet jacket, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG" + }, + { + "idx": 87, + "caption": "a bear is sitting in a tree branch", + "prompt": "a bear is sitting in a tree branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG" + }, + { + "idx": 88, + "caption": "a goat standing in the woods", + "prompt": "a goat standing in the woods, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG" + }, + { + "idx": 89, + "caption": "a military vehicle", + "prompt": "a military vehicle, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG" + }, + { + "idx": 90, + "caption": "a cup with a bunch of pencils in it", + "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG" + }, + { + "idx": 91, + "caption": "a jaguar in the water", + "prompt": "a jaguar in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG" + }, + { + "idx": 92, + "caption": "a flower made out of felt and felt", + "prompt": "a flower made out of felt and felt, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG" + }, + { + "idx": 93, + "caption": "a dog with a pink shirt on", + "prompt": "a dog with a pink shirt on, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG" + }, + { + "idx": 94, + "caption": "a close up of a bug on a rock", + "prompt": "a close up of a bug on a rock, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG" + }, + { + "idx": 95, + "caption": "a close up of a bird with a yellow eye", + "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG" + }, + { + "idx": 96, + "caption": "a hyen walking down a dirt road", + "prompt": "a hyen walking down a dirt road, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG" + }, + { + "idx": 97, + "caption": "a col of pictures of food including vegetables and vegetables", + "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG" + }, + { + "idx": 98, + "caption": "a brown barn", + "prompt": "a brown barn, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG" + }, + { + "idx": 99, + "caption": "a small black and white dog sitting on the ground", + "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG" + } + ] +} \ No newline at end of file diff --git a/benchmark/generator_runs/flux2_klein_9b/results.json b/benchmark/generator_runs/flux2_klein_9b/results.json new file mode 100644 index 0000000000000000000000000000000000000000..b400c3097c8737245d16d17e1ec057b2b863df37 --- /dev/null +++ b/benchmark/generator_runs/flux2_klein_9b/results.json @@ -0,0 +1,156 @@ +[ + { + "detector": "vib_net", + "name": "VIB-Net", + "auroc": 99.73, + "ap": 99.73, + "acc": 92.0, + "fake_detect_rate": 100.0, + "real_false_pos": 16.0, + "mean_p_fake_on_fakes": 0.9684, + "mean_p_fake_on_reals": 0.2277 + }, + { + "detector": "co_spy", + "name": "CO-SPY", + "auroc": 99.34, + "ap": 99.3, + "acc": 98.0, + "fake_detect_rate": 99.0, + "real_false_pos": 3.0, + "mean_p_fake_on_fakes": 0.8509, + "mean_p_fake_on_reals": 0.116 + }, + { + "detector": "pgc", + "name": "PGC", + "auroc": 97.35, + "ap": 96.45, + "acc": 53.5, + "fake_detect_rate": 7.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.4389, + "mean_p_fake_on_reals": 0.2672 + }, + { + "detector": "cure", + "name": "CuRe", + "auroc": 96.2, + "ap": 97.04, + "acc": 88.5, + "fake_detect_rate": 77.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.7393, + "mean_p_fake_on_reals": 0.0129 + }, + { + "detector": "aide", + "name": "AIDE", + "auroc": 93.73, + "ap": 88.94, + "acc": 81.0, + "fake_detect_rate": 69.0, + "real_false_pos": 7.0, + "mean_p_fake_on_fakes": 0.6246, + "mean_p_fake_on_reals": 0.1102 + }, + { + "detector": "safe", + "name": "SAFE", + "auroc": 92.32, + "ap": 84.09, + "acc": 88.5, + "fake_detect_rate": 87.0, + "real_false_pos": 10.0, + "mean_p_fake_on_fakes": 0.7988, + "mean_p_fake_on_reals": 0.1425 + }, + { + "detector": "npr", + "name": "NPR", + "auroc": 88.61, + "ap": 81.74, + "acc": 78.5, + "fake_detect_rate": 100.0, + "real_false_pos": 43.0, + "mean_p_fake_on_fakes": 0.9986, + "mean_p_fake_on_reals": 0.4365 + }, + { + "detector": "spai", + "name": "SPAI", + "auroc": 66.54, + "ap": 63.78, + "acc": 56.0, + "fake_detect_rate": 20.0, + "real_false_pos": 8.0, + "mean_p_fake_on_fakes": 0.2118, + "mean_p_fake_on_reals": 0.0778 + }, + { + "detector": "fat_former", + "name": "FatFormer", + "auroc": 45.86, + "ap": 46.65, + "acc": 50.5, + "fake_detect_rate": 5.0, + "real_false_pos": 4.0, + "mean_p_fake_on_fakes": 0.0477, + "mean_p_fake_on_reals": 0.0412 + }, + { + "detector": "dinov3", + "name": "DINOv3 probe", + "auroc": 45.76, + "ap": 45.57, + "acc": 46.0, + "fake_detect_rate": 5.0, + "real_false_pos": 13.0, + "mean_p_fake_on_fakes": 0.2405, + "mean_p_fake_on_reals": 0.2792 + }, + { + "detector": "adapted_dino", + "name": "Adapter-tuned DINOv2", + "auroc": 44.13, + "ap": 46.55, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.0, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "cnn", + "name": "CNNDetection (ResNet-50)", + "auroc": 40.86, + "ap": 46.29, + "acc": 50.0, + "fake_detect_rate": 1.0, + "real_false_pos": 1.0, + "mean_p_fake_on_fakes": 0.0116, + "mean_p_fake_on_reals": 0.0083 + }, + { + "detector": "dinov2", + "name": "DINOv2 probe", + "auroc": 35.29, + "ap": 41.64, + "acc": 46.0, + "fake_detect_rate": 5.0, + "real_false_pos": 13.0, + "mean_p_fake_on_fakes": 0.1056, + "mean_p_fake_on_reals": 0.1751 + }, + { + "detector": "clip", + "name": "UnivFD / CLIP", + "auroc": 26.07, + "ap": 36.57, + "acc": 47.0, + "fake_detect_rate": 1.0, + "real_false_pos": 7.0, + "mean_p_fake_on_fakes": 0.0715, + "mean_p_fake_on_reals": 0.1656 + } +] \ No newline at end of file diff --git a/benchmark/generator_runs/qwen_image_2_1/RESULTS.md b/benchmark/generator_runs/qwen_image_2_1/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..77a6d06d02353a39014f2b605cd3c96e1c21802b --- /dev/null +++ b/benchmark/generator_runs/qwen_image_2_1/RESULTS.md @@ -0,0 +1,20 @@ +# Detectors vs Qwen-Image-2.1 + +100 real images (people excluded) + 100 images generated by Qwen-Image-2.1 from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold. + +| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % | +|---|---|---|---|---|---|---| +| 1 | CO-SPY | 98.87 | 98.7 | 95.5 | 94.0 | 3.0 | +| 2 | PGC | 98.5 | 98.3 | 60.0 | 20.0 | 0.0 | +| 3 | VIB-Net | 98.35 | 98.51 | 91.5 | 98.0 | 15.0 | +| 4 | SPAI | 96.71 | 96.1 | 92.0 | 92.0 | 8.0 | +| 5 | CuRe | 92.68 | 94.28 | 82.0 | 64.0 | 0.0 | +| 6 | SAFE | 90.85 | 81.07 | 76.5 | 63.0 | 10.0 | +| 7 | NPR | 89.67 | 83.06 | 78.0 | 99.0 | 43.0 | +| 8 | AIDE | 89.42 | 82.41 | 60.0 | 27.0 | 7.0 | +| 9 | FatFormer | 76.74 | 78.17 | 67.5 | 39.0 | 4.0 | +| 10 | DINOv3 probe | 53.2 | 51.63 | 49.5 | 12.0 | 13.0 | +| 11 | CNNDetection (ResNet-50) | 52.52 | 52.58 | 50.0 | 1.0 | 1.0 | +| 12 | UnivFD / CLIP | 50.11 | 50.43 | 48.5 | 4.0 | 7.0 | +| 13 | Adapter-tuned DINOv2 | 49.15 | 49.17 | 50.0 | 0.0 | 0.0 | +| 14 | DINOv2 probe | 44.69 | 46.44 | 47.0 | 7.0 | 13.0 | diff --git a/benchmark/generator_runs/qwen_image_2_1/manifest.json b/benchmark/generator_runs/qwen_image_2_1/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..94075a7dadcc5e995e724580c8f9dd157f3e09a5 --- /dev/null +++ b/benchmark/generator_runs/qwen_image_2_1/manifest.json @@ -0,0 +1,611 @@ +{ + "label": "Qwen-Image-2.1", + "generator": "qwen_image_2_1", + "n": 100, + "res": 1024, + "settings": "1024px \u00b7 text-to-image from a caption", + "created": "2026-10-04", + "reuse_from": "benchmark/generator_runs/reference/manifest.json", + "pairs": [ + { + "idx": 0, + "caption": "sheep is white", + "prompt": "sheep is white, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG" + }, + { + "idx": 1, + "caption": "a bird sitting on a branch in a tree", + "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG" + }, + { + "idx": 2, + "caption": "a green grasshopper on a leaf", + "prompt": "a green grasshopper on a leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG" + }, + { + "idx": 3, + "caption": "the bell is black", + "prompt": "the bell is black, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG" + }, + { + "idx": 4, + "caption": "a car driving down a city street in the middle of the day", + "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png" + }, + { + "idx": 5, + "caption": "a dog is standing in the grass", + "prompt": "a dog is standing in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG" + }, + { + "idx": 6, + "caption": "a small kitten walking across a rocky beach", + "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG" + }, + { + "idx": 7, + "caption": "a mountain covered in snow", + "prompt": "a mountain covered in snow, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG" + }, + { + "idx": 8, + "caption": "a sailer sailing in the ocean near a lighthouse", + "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG" + }, + { + "idx": 9, + "caption": "a bowl of masoor", + "prompt": "a bowl of masoor, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG" + }, + { + "idx": 10, + "caption": "a green bird house", + "prompt": "a green bird house, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG" + }, + { + "idx": 11, + "caption": "a body of water", + "prompt": "a body of water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG" + }, + { + "idx": 12, + "caption": "a dog in a pool", + "prompt": "a dog in a pool, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG" + }, + { + "idx": 13, + "caption": "two tools, one with a wren and the other with a wren", + "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG" + }, + { + "idx": 14, + "caption": "a black leather folder with a red trim", + "prompt": "a black leather folder with a red trim, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png" + }, + { + "idx": 15, + "caption": "a small bird sitting on a wooden surface", + "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG" + }, + { + "idx": 16, + "caption": "mouse mouse on mouse mouse mouse", + "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG" + }, + { + "idx": 17, + "caption": "a small metal object on a wooden table", + "prompt": "a small metal object on a wooden table, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG" + }, + { + "idx": 18, + "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour", + "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG" + }, + { + "idx": 19, + "caption": "a white rabbit in a cage", + "prompt": "a white rabbit in a cage, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG" + }, + { + "idx": 20, + "caption": "a grass bug on a green leaf", + "prompt": "a grass bug on a green leaf, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG" + }, + { + "idx": 21, + "caption": "a field of grass", + "prompt": "a field of grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG" + }, + { + "idx": 22, + "caption": "a black padlock with a key", + "prompt": "a black padlock with a key, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png" + }, + { + "idx": 23, + "caption": "a group of balls", + "prompt": "a group of balls, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG" + }, + { + "idx": 24, + "caption": "three dogs standing on a hill", + "prompt": "three dogs standing on a hill, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG" + }, + { + "idx": 25, + "caption": "a dog standing in a field", + "prompt": "a dog standing in a field, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG" + }, + { + "idx": 26, + "caption": "a pink toilet", + "prompt": "a pink toilet, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG" + }, + { + "idx": 27, + "caption": "the sky is cloudy", + "prompt": "the sky is cloudy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG" + }, + { + "idx": 28, + "caption": "a bird is standing on the ground", + "prompt": "a bird is standing on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG" + }, + { + "idx": 29, + "caption": "a leopard sitting on a tree branch", + "prompt": "a leopard sitting on a tree branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG" + }, + { + "idx": 30, + "caption": "an old barn with a broken roof", + "prompt": "an old barn with a broken roof, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG" + }, + { + "idx": 31, + "caption": "a slug crawling on a concrete surface", + "prompt": "a slug crawling on a concrete surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG" + }, + { + "idx": 32, + "caption": "the three bottles of the serums are shown in the image", + "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png" + }, + { + "idx": 33, + "caption": "a bear cub", + "prompt": "a bear cub, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG" + }, + { + "idx": 34, + "caption": "a street with a yellow line painted on the road", + "prompt": "a street with a yellow line painted on the road, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png" + }, + { + "idx": 35, + "caption": "a lion fish swimming in the water", + "prompt": "a lion fish swimming in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG" + }, + { + "idx": 36, + "caption": "a plastic container with food", + "prompt": "a plastic container with food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png" + }, + { + "idx": 37, + "caption": "a car is driving down a street in a city", + "prompt": "a car is driving down a street in a city, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png" + }, + { + "idx": 38, + "caption": "a wooden desk", + "prompt": "a wooden desk, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG" + }, + { + "idx": 39, + "caption": "a bird standing in the water", + "prompt": "a bird standing in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG" + }, + { + "idx": 40, + "caption": "two parking meters", + "prompt": "two parking meters, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG" + }, + { + "idx": 41, + "caption": "a butterfly on a yellow flower", + "prompt": "a butterfly on a yellow flower, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG" + }, + { + "idx": 42, + "caption": "a green snake on a rock", + "prompt": "a green snake on a rock, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG" + }, + { + "idx": 43, + "caption": "a green snake in a tree", + "prompt": "a green snake in a tree, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG" + }, + { + "idx": 44, + "caption": "purple and gold beaded bracelet", + "prompt": "purple and gold beaded bracelet, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG" + }, + { + "idx": 45, + "caption": "a dog standing in the shade", + "prompt": "a dog standing in the shade, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG" + }, + { + "idx": 46, + "caption": "a cliff with a cliff face", + "prompt": "a cliff with a cliff face, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG" + }, + { + "idx": 47, + "caption": "a dog laying in the grass", + "prompt": "a dog laying in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG" + }, + { + "idx": 48, + "caption": "a small dog sitting on a bench", + "prompt": "a small dog sitting on a bench, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG" + }, + { + "idx": 49, + "caption": "a dog standing on a hill", + "prompt": "a dog standing on a hill, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG" + }, + { + "idx": 50, + "caption": "a white plate with food on it", + "prompt": "a white plate with food on it, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG" + }, + { + "idx": 51, + "caption": "a wooden chair", + "prompt": "a wooden chair, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG" + }, + { + "idx": 52, + "caption": "a large church with a stained glass window", + "prompt": "a large church with a stained glass window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG" + }, + { + "idx": 53, + "caption": "a dog running in the grass", + "prompt": "a dog running in the grass, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG" + }, + { + "idx": 54, + "caption": "a hand holding a hot dog with lete and lete", + "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png" + }, + { + "idx": 55, + "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish", + "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG" + }, + { + "idx": 56, + "caption": "a brown dog with a long tongue", + "prompt": "a brown dog with a long tongue, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG" + }, + { + "idx": 57, + "caption": "a wooden bench", + "prompt": "a wooden bench, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG" + }, + { + "idx": 58, + "caption": "a white dog toy", + "prompt": "a white dog toy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG" + }, + { + "idx": 59, + "caption": "a lake with trees and water in the background", + "prompt": "a lake with trees and water in the background, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG" + }, + { + "idx": 60, + "caption": "a crab crawling on the ground", + "prompt": "a crab crawling on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG" + }, + { + "idx": 61, + "caption": "a city street with a lot of buildings", + "prompt": "a city street with a lot of buildings, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png" + }, + { + "idx": 62, + "caption": "old wooden barrels stock photo, old wooden barrels by andy", + "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG" + }, + { + "idx": 63, + "caption": "a green beetle with red and green stripes", + "prompt": "a green beetle with red and green stripes, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG" + }, + { + "idx": 64, + "caption": "a plate of food", + "prompt": "a plate of food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG" + }, + { + "idx": 65, + "caption": "a fish in a tank", + "prompt": "a fish in a tank, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG" + }, + { + "idx": 66, + "caption": "a street with a lot of luggage and palm trees", + "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png" + }, + { + "idx": 67, + "caption": "a tree in the middle of a city", + "prompt": "a tree in the middle of a city, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png" + }, + { + "idx": 68, + "caption": "a kitchen with a large white island and a large window", + "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png" + }, + { + "idx": 69, + "caption": "a small brown and black dog standing on a white surface", + "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG" + }, + { + "idx": 70, + "caption": "a train on the tracks", + "prompt": "a train on the tracks, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG" + }, + { + "idx": 71, + "caption": "a large circular window", + "prompt": "a large circular window, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG" + }, + { + "idx": 72, + "caption": "a machine in a factory", + "prompt": "a machine in a factory, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG" + }, + { + "idx": 73, + "caption": "a red boat docked in a harbor", + "prompt": "a red boat docked in a harbor, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG" + }, + { + "idx": 74, + "caption": "a bird perched on a branch", + "prompt": "a bird perched on a branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG" + }, + { + "idx": 75, + "caption": "a large gray elephant", + "prompt": "a large gray elephant, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG" + }, + { + "idx": 76, + "caption": "a street with a star painted on the sidewalk", + "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png" + }, + { + "idx": 77, + "caption": "a fish in a tank with a fish inside", + "prompt": "a fish in a tank with a fish inside, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG" + }, + { + "idx": 78, + "caption": "a pot of soup", + "prompt": "a pot of soup, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG" + }, + { + "idx": 79, + "caption": "a bunch of bubbles", + "prompt": "a bunch of bubbles, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG" + }, + { + "idx": 80, + "caption": "a vase on display", + "prompt": "a vase on display, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG" + }, + { + "idx": 81, + "caption": "a toilet with a toilet seat and a toilet roll", + "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG" + }, + { + "idx": 82, + "caption": "a piece of food", + "prompt": "a piece of food, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG" + }, + { + "idx": 83, + "caption": "a car is parked in a street with a bus in the background", + "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png" + }, + { + "idx": 84, + "caption": "a white car driving down a street next to a building", + "prompt": "a white car driving down a street next to a building, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png" + }, + { + "idx": 85, + "caption": "a small canal in venice, italy", + "prompt": "a small canal in venice, italy, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG" + }, + { + "idx": 86, + "caption": "a black velvet jacket", + "prompt": "a black velvet jacket, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG" + }, + { + "idx": 87, + "caption": "a bear is sitting in a tree branch", + "prompt": "a bear is sitting in a tree branch, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG" + }, + { + "idx": 88, + "caption": "a goat standing in the woods", + "prompt": "a goat standing in the woods, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG" + }, + { + "idx": 89, + "caption": "a military vehicle", + "prompt": "a military vehicle, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG" + }, + { + "idx": 90, + "caption": "a cup with a bunch of pencils in it", + "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG" + }, + { + "idx": 91, + "caption": "a jaguar in the water", + "prompt": "a jaguar in the water, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG" + }, + { + "idx": 92, + "caption": "a flower made out of felt and felt", + "prompt": "a flower made out of felt and felt, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG" + }, + { + "idx": 93, + "caption": "a dog with a pink shirt on", + "prompt": "a dog with a pink shirt on, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG" + }, + { + "idx": 94, + "caption": "a close up of a bug on a rock", + "prompt": "a close up of a bug on a rock, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG" + }, + { + "idx": 95, + "caption": "a close up of a bird with a yellow eye", + "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG" + }, + { + "idx": 96, + "caption": "a hyen walking down a dirt road", + "prompt": "a hyen walking down a dirt road, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG" + }, + { + "idx": 97, + "caption": "a col of pictures of food including vegetables and vegetables", + "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG" + }, + { + "idx": 98, + "caption": "a brown barn", + "prompt": "a brown barn, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG" + }, + { + "idx": 99, + "caption": "a small black and white dog sitting on the ground", + "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail", + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG" + } + ] +} \ No newline at end of file diff --git a/benchmark/generator_runs/qwen_image_2_1/results.json b/benchmark/generator_runs/qwen_image_2_1/results.json new file mode 100644 index 0000000000000000000000000000000000000000..260cf435e1b3f8327b4e45b1313bf7bae9ebe611 --- /dev/null +++ b/benchmark/generator_runs/qwen_image_2_1/results.json @@ -0,0 +1,156 @@ +[ + { + "detector": "co_spy", + "name": "CO-SPY", + "auroc": 98.87, + "ap": 98.7, + "acc": 95.5, + "fake_detect_rate": 94.0, + "real_false_pos": 3.0, + "mean_p_fake_on_fakes": 0.8087, + "mean_p_fake_on_reals": 0.116 + }, + { + "detector": "pgc", + "name": "PGC", + "auroc": 98.5, + "ap": 98.3, + "acc": 60.0, + "fake_detect_rate": 20.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.4666, + "mean_p_fake_on_reals": 0.2672 + }, + { + "detector": "vib_net", + "name": "VIB-Net", + "auroc": 98.35, + "ap": 98.51, + "acc": 91.5, + "fake_detect_rate": 98.0, + "real_false_pos": 15.0, + "mean_p_fake_on_fakes": 0.9181, + "mean_p_fake_on_reals": 0.2277 + }, + { + "detector": "spai", + "name": "SPAI", + "auroc": 96.71, + "ap": 96.1, + "acc": 92.0, + "fake_detect_rate": 92.0, + "real_false_pos": 8.0, + "mean_p_fake_on_fakes": 0.8983, + "mean_p_fake_on_reals": 0.0778 + }, + { + "detector": "cure", + "name": "CuRe", + "auroc": 92.68, + "ap": 94.28, + "acc": 82.0, + "fake_detect_rate": 64.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.6284, + "mean_p_fake_on_reals": 0.0129 + }, + { + "detector": "safe", + "name": "SAFE", + "auroc": 90.85, + "ap": 81.07, + "acc": 76.5, + "fake_detect_rate": 63.0, + "real_false_pos": 10.0, + "mean_p_fake_on_fakes": 0.6334, + "mean_p_fake_on_reals": 0.1425 + }, + { + "detector": "npr", + "name": "NPR", + "auroc": 89.67, + "ap": 83.06, + "acc": 78.0, + "fake_detect_rate": 99.0, + "real_false_pos": 43.0, + "mean_p_fake_on_fakes": 0.9944, + "mean_p_fake_on_reals": 0.4365 + }, + { + "detector": "aide", + "name": "AIDE", + "auroc": 89.42, + "ap": 82.41, + "acc": 60.0, + "fake_detect_rate": 27.0, + "real_false_pos": 7.0, + "mean_p_fake_on_fakes": 0.3517, + "mean_p_fake_on_reals": 0.1102 + }, + { + "detector": "fat_former", + "name": "FatFormer", + "auroc": 76.74, + "ap": 78.17, + "acc": 67.5, + "fake_detect_rate": 39.0, + "real_false_pos": 4.0, + "mean_p_fake_on_fakes": 0.3841, + "mean_p_fake_on_reals": 0.0412 + }, + { + "detector": "dinov3", + "name": "DINOv3 probe", + "auroc": 53.2, + "ap": 51.63, + "acc": 49.5, + "fake_detect_rate": 12.0, + "real_false_pos": 13.0, + "mean_p_fake_on_fakes": 0.2902, + "mean_p_fake_on_reals": 0.2792 + }, + { + "detector": "cnn", + "name": "CNNDetection (ResNet-50)", + "auroc": 52.52, + "ap": 52.58, + "acc": 50.0, + "fake_detect_rate": 1.0, + "real_false_pos": 1.0, + "mean_p_fake_on_fakes": 0.0126, + "mean_p_fake_on_reals": 0.0083 + }, + { + "detector": "clip", + "name": "UnivFD / CLIP", + "auroc": 50.11, + "ap": 50.43, + "acc": 48.5, + "fake_detect_rate": 4.0, + "real_false_pos": 7.0, + "mean_p_fake_on_fakes": 0.161, + "mean_p_fake_on_reals": 0.1656 + }, + { + "detector": "adapted_dino", + "name": "Adapter-tuned DINOv2", + "auroc": 49.15, + "ap": 49.17, + "acc": 50.0, + "fake_detect_rate": 0.0, + "real_false_pos": 0.0, + "mean_p_fake_on_fakes": 0.0, + "mean_p_fake_on_reals": 0.0 + }, + { + "detector": "dinov2", + "name": "DINOv2 probe", + "auroc": 44.69, + "ap": 46.44, + "acc": 47.0, + "fake_detect_rate": 7.0, + "real_false_pos": 13.0, + "mean_p_fake_on_fakes": 0.1279, + "mean_p_fake_on_reals": 0.1751 + } +] \ No newline at end of file diff --git a/benchmark/generator_runs/reference/manifest.json b/benchmark/generator_runs/reference/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..eccc361fbb657356fc27df5216b89e600bdf2286 --- /dev/null +++ b/benchmark/generator_runs/reference/manifest.json @@ -0,0 +1,609 @@ +{ + "n": 100, + "steps": 28, + "res": 512, + "attempts": 271, + "pairs": [ + { + "idx": 0, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG", + "caption": "sheep is white", + "prompt": "sheep is white, photorealistic, high detail" + }, + { + "idx": 1, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG", + "caption": "a bird sitting on a branch in a tree", + "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail" + }, + { + "idx": 2, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG", + "caption": "a green grasshopper on a leaf", + "prompt": "a green grasshopper on a leaf, photorealistic, high detail" + }, + { + "idx": 3, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG", + "caption": "the bell is black", + "prompt": "the bell is black, photorealistic, high detail" + }, + { + "idx": 4, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png", + "caption": "a car driving down a city street in the middle of the day", + "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail" + }, + { + "idx": 5, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG", + "caption": "a dog is standing in the grass", + "prompt": "a dog is standing in the grass, photorealistic, high detail" + }, + { + "idx": 6, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG", + "caption": "a small kitten walking across a rocky beach", + "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail" + }, + { + "idx": 7, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG", + "caption": "a mountain covered in snow", + "prompt": "a mountain covered in snow, photorealistic, high detail" + }, + { + "idx": 8, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG", + "caption": "a sailer sailing in the ocean near a lighthouse", + "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail" + }, + { + "idx": 9, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG", + "caption": "a bowl of masoor", + "prompt": "a bowl of masoor, photorealistic, high detail" + }, + { + "idx": 10, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG", + "caption": "a green bird house", + "prompt": "a green bird house, photorealistic, high detail" + }, + { + "idx": 11, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG", + "caption": "a body of water", + "prompt": "a body of water, photorealistic, high detail" + }, + { + "idx": 12, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG", + "caption": "a dog in a pool", + "prompt": "a dog in a pool, photorealistic, high detail" + }, + { + "idx": 13, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG", + "caption": "two tools, one with a wren and the other with a wren", + "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail" + }, + { + "idx": 14, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png", + "caption": "a black leather folder with a red trim", + "prompt": "a black leather folder with a red trim, photorealistic, high detail" + }, + { + "idx": 15, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG", + "caption": "a small bird sitting on a wooden surface", + "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail" + }, + { + "idx": 16, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG", + "caption": "mouse mouse on mouse mouse mouse", + "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail" + }, + { + "idx": 17, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG", + "caption": "a small metal object on a wooden table", + "prompt": "a small metal object on a wooden table, photorealistic, high detail" + }, + { + "idx": 18, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG", + "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour", + "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail" + }, + { + "idx": 19, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG", + "caption": "a white rabbit in a cage", + "prompt": "a white rabbit in a cage, photorealistic, high detail" + }, + { + "idx": 20, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG", + "caption": "a grass bug on a green leaf", + "prompt": "a grass bug on a green leaf, photorealistic, high detail" + }, + { + "idx": 21, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG", + "caption": "a field of grass", + "prompt": "a field of grass, photorealistic, high detail" + }, + { + "idx": 22, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png", + "caption": "a black padlock with a key", + "prompt": "a black padlock with a key, photorealistic, high detail" + }, + { + "idx": 23, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG", + "caption": "a group of balls", + "prompt": "a group of balls, photorealistic, high detail" + }, + { + "idx": 24, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG", + "caption": "three dogs standing on a hill", + "prompt": "three dogs standing on a hill, photorealistic, high detail" + }, + { + "idx": 25, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG", + "caption": "a dog standing in a field", + "prompt": "a dog standing in a field, photorealistic, high detail" + }, + { + "idx": 26, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG", + "caption": "a pink toilet", + "prompt": "a pink toilet, photorealistic, high detail" + }, + { + "idx": 27, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG", + "caption": "the sky is cloudy", + "prompt": "the sky is cloudy, photorealistic, high detail" + }, + { + "idx": 28, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG", + "caption": "a bird is standing on the ground", + "prompt": "a bird is standing on the ground, photorealistic, high detail" + }, + { + "idx": 29, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG", + "caption": "a leopard sitting on a tree branch", + "prompt": "a leopard sitting on a tree branch, photorealistic, high detail" + }, + { + "idx": 30, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG", + "caption": "an old barn with a broken roof", + "prompt": "an old barn with a broken roof, photorealistic, high detail" + }, + { + "idx": 31, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG", + "caption": "a slug crawling on a concrete surface", + "prompt": "a slug crawling on a concrete surface, photorealistic, high detail" + }, + { + "idx": 32, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png", + "caption": "the three bottles of the serums are shown in the image", + "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail" + }, + { + "idx": 33, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG", + "caption": "a bear cub", + "prompt": "a bear cub, photorealistic, high detail" + }, + { + "idx": 34, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png", + "caption": "a street with a yellow line painted on the road", + "prompt": "a street with a yellow line painted on the road, photorealistic, high detail" + }, + { + "idx": 35, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG", + "caption": "a lion fish swimming in the water", + "prompt": "a lion fish swimming in the water, photorealistic, high detail" + }, + { + "idx": 36, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png", + "caption": "a plastic container with food", + "prompt": "a plastic container with food, photorealistic, high detail" + }, + { + "idx": 37, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png", + "caption": "a car is driving down a street in a city", + "prompt": "a car is driving down a street in a city, photorealistic, high detail" + }, + { + "idx": 38, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG", + "caption": "a wooden desk", + "prompt": "a wooden desk, photorealistic, high detail" + }, + { + "idx": 39, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG", + "caption": "a bird standing in the water", + "prompt": "a bird standing in the water, photorealistic, high detail" + }, + { + "idx": 40, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG", + "caption": "two parking meters", + "prompt": "two parking meters, photorealistic, high detail" + }, + { + "idx": 41, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG", + "caption": "a butterfly on a yellow flower", + "prompt": "a butterfly on a yellow flower, photorealistic, high detail" + }, + { + "idx": 42, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG", + "caption": "a green snake on a rock", + "prompt": "a green snake on a rock, photorealistic, high detail" + }, + { + "idx": 43, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG", + "caption": "a green snake in a tree", + "prompt": "a green snake in a tree, photorealistic, high detail" + }, + { + "idx": 44, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG", + "caption": "purple and gold beaded bracelet", + "prompt": "purple and gold beaded bracelet, photorealistic, high detail" + }, + { + "idx": 45, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG", + "caption": "a dog standing in the shade", + "prompt": "a dog standing in the shade, photorealistic, high detail" + }, + { + "idx": 46, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG", + "caption": "a cliff with a cliff face", + "prompt": "a cliff with a cliff face, photorealistic, high detail" + }, + { + "idx": 47, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG", + "caption": "a dog laying in the grass", + "prompt": "a dog laying in the grass, photorealistic, high detail" + }, + { + "idx": 48, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG", + "caption": "a small dog sitting on a bench", + "prompt": "a small dog sitting on a bench, photorealistic, high detail" + }, + { + "idx": 49, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG", + "caption": "a dog standing on a hill", + "prompt": "a dog standing on a hill, photorealistic, high detail" + }, + { + "idx": 50, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG", + "caption": "a white plate with food on it", + "prompt": "a white plate with food on it, photorealistic, high detail" + }, + { + "idx": 51, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG", + "caption": "a wooden chair", + "prompt": "a wooden chair, photorealistic, high detail" + }, + { + "idx": 52, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG", + "caption": "a large church with a stained glass window", + "prompt": "a large church with a stained glass window, photorealistic, high detail" + }, + { + "idx": 53, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG", + "caption": "a dog running in the grass", + "prompt": "a dog running in the grass, photorealistic, high detail" + }, + { + "idx": 54, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png", + "caption": "a hand holding a hot dog with lete and lete", + "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail" + }, + { + "idx": 55, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG", + "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish", + "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail" + }, + { + "idx": 56, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG", + "caption": "a brown dog with a long tongue", + "prompt": "a brown dog with a long tongue, photorealistic, high detail" + }, + { + "idx": 57, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG", + "caption": "a wooden bench", + "prompt": "a wooden bench, photorealistic, high detail" + }, + { + "idx": 58, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG", + "caption": "a white dog toy", + "prompt": "a white dog toy, photorealistic, high detail" + }, + { + "idx": 59, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG", + "caption": "a lake with trees and water in the background", + "prompt": "a lake with trees and water in the background, photorealistic, high detail" + }, + { + "idx": 60, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG", + "caption": "a crab crawling on the ground", + "prompt": "a crab crawling on the ground, photorealistic, high detail" + }, + { + "idx": 61, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png", + "caption": "a city street with a lot of buildings", + "prompt": "a city street with a lot of buildings, photorealistic, high detail" + }, + { + "idx": 62, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG", + "caption": "old wooden barrels stock photo, old wooden barrels by andy", + "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail" + }, + { + "idx": 63, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG", + "caption": "a green beetle with red and green stripes", + "prompt": "a green beetle with red and green stripes, photorealistic, high detail" + }, + { + "idx": 64, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG", + "caption": "a plate of food", + "prompt": "a plate of food, photorealistic, high detail" + }, + { + "idx": 65, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG", + "caption": "a fish in a tank", + "prompt": "a fish in a tank, photorealistic, high detail" + }, + { + "idx": 66, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png", + "caption": "a street with a lot of luggage and palm trees", + "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail" + }, + { + "idx": 67, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png", + "caption": "a tree in the middle of a city", + "prompt": "a tree in the middle of a city, photorealistic, high detail" + }, + { + "idx": 68, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png", + "caption": "a kitchen with a large white island and a large window", + "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail" + }, + { + "idx": 69, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG", + "caption": "a small brown and black dog standing on a white surface", + "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail" + }, + { + "idx": 70, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG", + "caption": "a train on the tracks", + "prompt": "a train on the tracks, photorealistic, high detail" + }, + { + "idx": 71, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG", + "caption": "a large circular window", + "prompt": "a large circular window, photorealistic, high detail" + }, + { + "idx": 72, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG", + "caption": "a machine in a factory", + "prompt": "a machine in a factory, photorealistic, high detail" + }, + { + "idx": 73, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG", + "caption": "a red boat docked in a harbor", + "prompt": "a red boat docked in a harbor, photorealistic, high detail" + }, + { + "idx": 74, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG", + "caption": "a bird perched on a branch", + "prompt": "a bird perched on a branch, photorealistic, high detail" + }, + { + "idx": 75, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG", + "caption": "a large gray elephant", + "prompt": "a large gray elephant, photorealistic, high detail" + }, + { + "idx": 76, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png", + "caption": "a street with a star painted on the sidewalk", + "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail" + }, + { + "idx": 77, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG", + "caption": "a fish in a tank with a fish inside", + "prompt": "a fish in a tank with a fish inside, photorealistic, high detail" + }, + { + "idx": 78, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG", + "caption": "a pot of soup", + "prompt": "a pot of soup, photorealistic, high detail" + }, + { + "idx": 79, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG", + "caption": "a bunch of bubbles", + "prompt": "a bunch of bubbles, photorealistic, high detail" + }, + { + "idx": 80, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG", + "caption": "a vase on display", + "prompt": "a vase on display, photorealistic, high detail" + }, + { + "idx": 81, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG", + "caption": "a toilet with a toilet seat and a toilet roll", + "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail" + }, + { + "idx": 82, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG", + "caption": "a piece of food", + "prompt": "a piece of food, photorealistic, high detail" + }, + { + "idx": 83, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png", + "caption": "a car is parked in a street with a bus in the background", + "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail" + }, + { + "idx": 84, + "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png", + "caption": "a white car driving down a street next to a building", + "prompt": "a white car driving down a street next to a building, photorealistic, high detail" + }, + { + "idx": 85, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG", + "caption": "a small canal in venice, italy", + "prompt": "a small canal in venice, italy, photorealistic, high detail" + }, + { + "idx": 86, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG", + "caption": "a black velvet jacket", + "prompt": "a black velvet jacket, photorealistic, high detail" + }, + { + "idx": 87, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG", + "caption": "a bear is sitting in a tree branch", + "prompt": "a bear is sitting in a tree branch, photorealistic, high detail" + }, + { + "idx": 88, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG", + "caption": "a goat standing in the woods", + "prompt": "a goat standing in the woods, photorealistic, high detail" + }, + { + "idx": 89, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG", + "caption": "a military vehicle", + "prompt": "a military vehicle, photorealistic, high detail" + }, + { + "idx": 90, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG", + "caption": "a cup with a bunch of pencils in it", + "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail" + }, + { + "idx": 91, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG", + "caption": "a jaguar in the water", + "prompt": "a jaguar in the water, photorealistic, high detail" + }, + { + "idx": 92, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG", + "caption": "a flower made out of felt and felt", + "prompt": "a flower made out of felt and felt, photorealistic, high detail" + }, + { + "idx": 93, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG", + "caption": "a dog with a pink shirt on", + "prompt": "a dog with a pink shirt on, photorealistic, high detail" + }, + { + "idx": 94, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG", + "caption": "a close up of a bug on a rock", + "prompt": "a close up of a bug on a rock, photorealistic, high detail" + }, + { + "idx": 95, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG", + "caption": "a close up of a bird with a yellow eye", + "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail" + }, + { + "idx": 96, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG", + "caption": "a hyen walking down a dirt road", + "prompt": "a hyen walking down a dirt road, photorealistic, high detail" + }, + { + "idx": 97, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG", + "caption": "a col of pictures of food including vegetables and vegetables", + "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail" + }, + { + "idx": 98, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG", + "caption": "a brown barn", + "prompt": "a brown barn, photorealistic, high detail" + }, + { + "idx": 99, + "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG", + "caption": "a small black and white dog sitting on the ground", + "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail" + } + ], + "label": "Ideogram 4.0 (512px)" +} \ No newline at end of file diff --git a/benchmark/leaderboard.py b/benchmark/leaderboard.py new file mode 100644 index 0000000000000000000000000000000000000000..7e026df6fe1c035739955187271105e33b2b3e6c --- /dev/null +++ b/benchmark/leaderboard.py @@ -0,0 +1,160 @@ +"""Validate benchmark submissions and render the leaderboard. + + python benchmark/leaderboard.py # write LEADERBOARD.md + python benchmark/leaderboard.py --check # CI: non-zero exit if invalid or stale + +Output is deterministic (no timestamps) so ``--check`` is a stable up-to-date test. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +BENCH_DIR = Path(__file__).resolve().parent +SUBMISSIONS_DIR = BENCH_DIR / "submissions" +LEADERBOARD_MD = BENCH_DIR.parent / "LEADERBOARD.md" + +# Track order, titles, and the metric each track ranks by. +TRACKS = { + "forensynths": { + "title": "ForenSynths — image (ProGAN-trained, cross-generator)", + "primary": "map", "modality": "image", + }, + "genimage-sdv1.4": { + "title": "GenImage SD-v1.4 — image", "primary": "auroc", "modality": "image", + }, + "genvideo-val": { + "title": "GenVideo-Val — video", "primary": "ap", "modality": "video", + }, +} +METRIC_COLS = [("acc", "Acc"), ("map", "mAP"), ("ap", "AP"), ("auroc", "AUROC")] +EVALUATIONS = {"official-weights", "from-scratch", "frozen-probe", "training-free", "external"} +MODALITIES = {"image", "video"} +REQUIRED = ["method", "track", "modality", "evaluation", "metrics", "submitted_by"] + + +def validate(sub: dict, where: str) -> list[str]: + """Return a list of human-readable errors for one submission (empty = valid).""" + errs = [] + for k in REQUIRED: + if k not in sub: + errs.append(f"missing required field '{k}'") + if sub.get("track") not in TRACKS: + errs.append(f"track '{sub.get('track')}' not in {sorted(TRACKS)}") + if sub.get("modality") not in MODALITIES: + errs.append(f"modality '{sub.get('modality')}' not in {sorted(MODALITIES)}") + if sub.get("evaluation") not in EVALUATIONS: + errs.append(f"evaluation '{sub.get('evaluation')}' not in {sorted(EVALUATIONS)}") + metrics = sub.get("metrics") + if not isinstance(metrics, dict): + errs.append("'metrics' must be an object") + else: + vals = [metrics.get(m) for m, _ in METRIC_COLS] + if not any(isinstance(v, (int, float)) for v in vals): + errs.append("'metrics' needs at least one numeric value among acc/map/ap/auroc") + return [f"{where}: {e}" for e in errs] + + +def load_submissions(): + """Return (valid_submissions, errors).""" + subs, errors = [], [] + for path in sorted(SUBMISSIONS_DIR.glob("*.json")): + try: + sub = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + errors.append(f"{path.name}: invalid JSON ({exc})") + continue + errs = validate(sub, path.name) + if errs: + errors.extend(errs) + continue + sub["_file"] = path.name + subs.append(sub) + return subs, errors + + +def _fmt(v) -> str: + return f"{v:.2f}" if isinstance(v, (int, float)) else "—" + + +def _sort_key(sub, primary): + v = sub.get("metrics", {}).get(primary) + # rank desc by primary; missing primary sorts last + return (0, -v) if isinstance(v, (int, float)) else (1, 0.0) + + +def render() -> str: + subs, errors = load_submissions() + lines = [ + "# SynthScope Leaderboard (v1)", "", + "_Auto-generated by `benchmark/leaderboard.py` — do not edit by hand._ " + "See [benchmark/README.md](benchmark/README.md) for the protocol and how to submit.", + "", "This is the **standardized, versioned** table: a few tracks with a fixed protocol that anyone can submit to. The web page's **Leaderboard** tab is broader (every paper benchmark, 30+ groups) and the repo supports many more datasets, which download and organize themselves: see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).", "", + "Metrics are percentages. **mAP** = mean AP over generators. Each track is ranked by " + "its primary metric (in bold).", "", + ] + for track_id, cfg in TRACKS.items(): + rows = [s for s in subs if s.get("track") == track_id] + primary = cfg["primary"] + header_cells = ["Rank", "Method", "Eval"] + for key, label in METRIC_COLS: + header_cells.append(f"**{label}**" if key == primary else label) + header_cells += ["Verified", "Notes"] + lines.append(f"## {cfg['title']}") + lines.append("") + if not rows: + lines += ["_No submissions yet._", ""] + continue + rows.sort(key=lambda s: _sort_key(s, primary)) + lines.append("| " + " | ".join(header_cells) + " |") + lines.append("| " + " | ".join(["---"] * len(header_cells)) + " |") + for i, s in enumerate(rows, 1): + m = s.get("metrics", {}) + cells = [str(i), s["method"], s["evaluation"]] + cells += [_fmt(m.get(key)) for key, _ in METRIC_COLS] + cells.append("✓" if s.get("verified") else "") + note = s.get("protocol_notes") or "" + cells.append(note.replace("|", "/")) + lines.append("| " + " | ".join(cells) + " |") + lines.append("") + if errors: + lines += ["## ⚠️ Invalid submissions", ""] + [f"- {e}" for e in errors] + [""] + return "\n".join(lines).rstrip() + "\n" + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Build/validate the SynthScope leaderboard.") + ap.add_argument("--check", action="store_true", + help="validate submissions and verify LEADERBOARD.md is up to date (CI)") + args = ap.parse_args(argv) + + subs, errors = load_submissions() + content = render() + + if args.check: + ok = True + if errors: + print("Invalid submissions:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + ok = False + current = LEADERBOARD_MD.read_text() if LEADERBOARD_MD.exists() else "" + if current != content: + print("LEADERBOARD.md is stale — run `python benchmark/leaderboard.py`.", file=sys.stderr) + ok = False + if ok: + print(f"OK: {len(subs)} submissions, leaderboard up to date.") + return 0 if ok else 1 + + LEADERBOARD_MD.write_text(content) + print(f"Wrote {LEADERBOARD_MD} ({len(subs)} submissions" + + (f", {len(errors)} invalid" if errors else "") + ").") + for e in errors: + print(f" invalid: {e}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/schema.json b/benchmark/schema.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba842e3dc4f6265748be90ccc12fbae979f8da3 --- /dev/null +++ b/benchmark/schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SynthScope benchmark submission (v1)", + "description": "One result for one (method, track). Lives in benchmark/submissions/*.json.", + "type": "object", + "additionalProperties": false, + "required": ["method", "track", "modality", "evaluation", "metrics", "submitted_by"], + "properties": { + "method": {"type": "string", "description": "Human-readable method name."}, + "track": {"type": "string", "enum": ["forensynths", "genimage-sdv1.4", "genvideo-val"]}, + "modality": {"type": "string", "enum": ["image", "video"]}, + "model_type": {"type": ["string", "null"], "description": "SynthScope detector key, if applicable."}, + "paper": {"type": ["string", "null"]}, + "url": {"type": ["string", "null"], "description": "Paper or code URL."}, + "evaluation": { + "type": "string", + "enum": ["official-weights", "from-scratch", "frozen-probe", "training-free", "external"] + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "description": "Percentages (0-100). At least one must be non-null.", + "properties": { + "acc": {"type": ["number", "null"]}, + "map": {"type": ["number", "null"], "description": "mean AP over generators"}, + "ap": {"type": ["number", "null"]}, + "auroc": {"type": ["number", "null"]} + } + }, + "per_generator": { + "type": ["object", "null"], + "description": "Optional per-generator metric breakdown." + }, + "submitted_by": {"type": "string"}, + "date": {"type": ["string", "null"], "description": "YYYY-MM-DD."}, + "verified": {"type": "boolean", "description": "Traces to a logged SynthScope evaluation."}, + "source": {"type": ["string", "null"], "description": "Log / run path or external reference."}, + "protocol_notes": {"type": ["string", "null"]}, + "synthscope_version": {"type": ["string", "null"]} + } +} diff --git a/benchmark/seed_reference_results.py b/benchmark/seed_reference_results.py new file mode 100644 index 0000000000000000000000000000000000000000..73e106609a0d0efb27263d2eb9d260d9544d9f1c --- /dev/null +++ b/benchmark/seed_reference_results.py @@ -0,0 +1,70 @@ +"""Generate the reference (already-reproduced) leaderboard submissions. + +Every number here is **transcribed from the verified reproduction table in the repo README** +(each traces to a logged `Test Metrics` evaluation; see `repro/FINAL_RESULTS.md`). No number is +estimated. Re-run to regenerate `submissions/*.json`: + + python benchmark/seed_reference_results.py +""" +import json +from pathlib import Path + +SUB = Path(__file__).resolve().parent / "submissions" +REPO = "https://github.com/SMSD75/SynthScope" +SOURCE = "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md" +DATE = "2026-06-02" + +# (slug, method, track, modality, model_type, paper, evaluation, metrics, protocol_notes) +ROWS = [ + # --- ForenSynths (image) --- + ("fat_former", "FatFormer", "forensynths", "image", "fat_former", "Liu et al., CVPR'24", + "official-weights", {"acc": 98.39, "map": 99.70}, "ForenSynths-8 subset; official released weights"), + ("co_spy_official", "CO-SPY (official weights)", "forensynths", "image", "co_spy", "Cheng et al., CVPR'25", + "official-weights", {"acc": 95.12, "map": 98.71}, "ForenSynths-8; vendored official architecture"), + ("npr", "NPR", "forensynths", "image", "npr", "Tan et al., CVPR'24", + "official-weights", {"acc": 91.83, "map": 87.09}, "ForenSynths-8; official ProGAN weights"), + ("aide", "AIDE", "forensynths", "image", "aide", "Yan et al., ICLR'25", + "official-weights", {"map": 92.08}, "ForenSynths-8; mAP reproduces, acc@0.5 miscalibrated (single-view vs official 5-view DCT)"), + ("clip", "UnivFD / CLIP", "forensynths", "image", "clip", "Ojha et al., CVPR'23", + "frozen-probe", {"map": 97.2, "auroc": 97.0}, "ForenSynths; frozen CLIP ViT-L/14 probe (ProGAN)"), + ("cnn", "CNNDetection", "forensynths", "image", "cnn", "Wang et al., CVPR'20", + "from-scratch", {"map": 96.8, "auroc": 96.6}, "ForenSynths; ResNet-50 fine-tune (ProGAN)"), + ("dinov2", "DINOv2 probe", "forensynths", "image", "dinov2", "Oquab et al., 2023", + "frozen-probe", {"map": 95.03, "auroc": 94.86}, "ForenSynths; frozen DINOv2 ViT-L/14 probe"), + ("safe", "SAFE", "forensynths", "image", "safe", "Li et al., KDD'25", + "from-scratch", {"map": 94.13, "auroc": 93.68}, "ForenSynths; trained from scratch"), + ("co_spy_scratch", "CO-SPY (from scratch)", "forensynths", "image", "co_spy", "Cheng et al., CVPR'25", + "from-scratch", {"map": 96.98, "auroc": 96.79}, "ForenSynths; SynthScope re-implementation trained on ProGAN"), + # --- GenImage SD-v1.4 (image) --- + ("spai_official", "SPAI (official weights)", "genimage-sdv1.4", "image", "spai", "Karageorgiou et al., CVPR'25", + "official-weights", {"auroc": 98.45, "ap": 94.01}, "GenImage SD-v1.4; vendored official architecture, fixed-res 2x2x224"), + ("vib_net", "VIB-Net", "genimage-sdv1.4", "image", "vib_net", "Zhang et al., CVPR'25", + "from-scratch", {"map": 99.33, "auroc": 99.33}, "GenImage SD-v1.4 single real/fake pool (mAP == AUROC by construction)"), + ("spai_scratch", "SPAI (from scratch)", "genimage-sdv1.4", "image", "spai", "Karageorgiou et al., CVPR'25", + "from-scratch", {"auroc": 97.66}, "DMID Latent-Diffusion vs COCO; SynthScope fixed-resolution variant"), + # --- GenVideo-Val (video) --- + ("d3", "D3", "genvideo-val", "video", "d3", "ICCV'25", + "training-free", {"ap": 95.6, "auroc": 93.6}, "Training-free frozen CLIP 2nd-order trajectory"), + ("restrav", "ReStraV", "genvideo-val", "video", "restrav", "2025", + "frozen-probe", {"ap": 96.67, "auroc": 97.06}, "Frozen DINOv2 trajectory geometry + MLP"), +] + +METRIC_KEYS = ["acc", "map", "ap", "auroc"] + + +def main() -> None: + SUB.mkdir(parents=True, exist_ok=True) + for slug, method, track, modality, model_type, paper, evaluation, metrics, notes in ROWS: + full_metrics = {k: metrics.get(k) for k in METRIC_KEYS} + sub = { + "method": method, "track": track, "modality": modality, "model_type": model_type, + "paper": paper, "url": REPO, "evaluation": evaluation, "metrics": full_metrics, + "per_generator": None, "submitted_by": "SynthScope", "date": DATE, + "verified": True, "source": SOURCE, "protocol_notes": notes, "synthscope_version": "0.1.0", + } + (SUB / f"{slug}.json").write_text(json.dumps(sub, indent=2) + "\n") + print(f"Wrote {len(ROWS)} reference submissions to {SUB}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/submissions/aide.json b/benchmark/submissions/aide.json new file mode 100644 index 0000000000000000000000000000000000000000..5c97e1d5b59dfbbe12f62e31614e933d29a98cb1 --- /dev/null +++ b/benchmark/submissions/aide.json @@ -0,0 +1,22 @@ +{ + "method": "AIDE", + "track": "forensynths", + "modality": "image", + "model_type": "aide", + "paper": "Yan et al., ICLR'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "official-weights", + "metrics": { + "acc": null, + "map": 92.08, + "ap": null, + "auroc": null + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths-8; mAP reproduces, acc@0.5 miscalibrated (single-view vs official 5-view DCT)", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/clip.json b/benchmark/submissions/clip.json new file mode 100644 index 0000000000000000000000000000000000000000..6fa0e7423d955bb161c5b75ec0e707750057eeb4 --- /dev/null +++ b/benchmark/submissions/clip.json @@ -0,0 +1,22 @@ +{ + "method": "UnivFD / CLIP", + "track": "forensynths", + "modality": "image", + "model_type": "clip", + "paper": "Ojha et al., CVPR'23", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "frozen-probe", + "metrics": { + "acc": null, + "map": 97.2, + "ap": null, + "auroc": 97.0 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths; frozen CLIP ViT-L/14 probe (ProGAN)", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/cnn.json b/benchmark/submissions/cnn.json new file mode 100644 index 0000000000000000000000000000000000000000..53cad9a9fe268579c3ec6179a999ec40144ecace --- /dev/null +++ b/benchmark/submissions/cnn.json @@ -0,0 +1,22 @@ +{ + "method": "CNNDetection", + "track": "forensynths", + "modality": "image", + "model_type": "cnn", + "paper": "Wang et al., CVPR'20", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "from-scratch", + "metrics": { + "acc": null, + "map": 96.8, + "ap": null, + "auroc": 96.6 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths; ResNet-50 fine-tune (ProGAN)", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/co_spy_official.json b/benchmark/submissions/co_spy_official.json new file mode 100644 index 0000000000000000000000000000000000000000..37b9d4960f4c33cf58fea9ffae05017641667624 --- /dev/null +++ b/benchmark/submissions/co_spy_official.json @@ -0,0 +1,22 @@ +{ + "method": "CO-SPY (official weights)", + "track": "forensynths", + "modality": "image", + "model_type": "co_spy", + "paper": "Cheng et al., CVPR'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "official-weights", + "metrics": { + "acc": 95.12, + "map": 98.71, + "ap": null, + "auroc": null + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths-8; vendored official architecture", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/co_spy_scratch.json b/benchmark/submissions/co_spy_scratch.json new file mode 100644 index 0000000000000000000000000000000000000000..3f9e100716e2f91cba860cab4eca7bef0a136c74 --- /dev/null +++ b/benchmark/submissions/co_spy_scratch.json @@ -0,0 +1,22 @@ +{ + "method": "CO-SPY (from scratch)", + "track": "forensynths", + "modality": "image", + "model_type": "co_spy", + "paper": "Cheng et al., CVPR'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "from-scratch", + "metrics": { + "acc": null, + "map": 96.98, + "ap": null, + "auroc": 96.79 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths; SynthScope re-implementation trained on ProGAN", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/d3.json b/benchmark/submissions/d3.json new file mode 100644 index 0000000000000000000000000000000000000000..0fb57e4e3695654bf69611f3f010369299171d37 --- /dev/null +++ b/benchmark/submissions/d3.json @@ -0,0 +1,22 @@ +{ + "method": "D3", + "track": "genvideo-val", + "modality": "video", + "model_type": "d3", + "paper": "ICCV'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "training-free", + "metrics": { + "acc": null, + "map": null, + "ap": 95.6, + "auroc": 93.6 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "Training-free frozen CLIP 2nd-order trajectory", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/dinov2.json b/benchmark/submissions/dinov2.json new file mode 100644 index 0000000000000000000000000000000000000000..5e1c8783e762d10693cd94b98e3e8efceae035cd --- /dev/null +++ b/benchmark/submissions/dinov2.json @@ -0,0 +1,22 @@ +{ + "method": "DINOv2 probe", + "track": "forensynths", + "modality": "image", + "model_type": "dinov2", + "paper": "Oquab et al., 2023", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "frozen-probe", + "metrics": { + "acc": null, + "map": 95.03, + "ap": null, + "auroc": 94.86 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths; frozen DINOv2 ViT-L/14 probe", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/fat_former.json b/benchmark/submissions/fat_former.json new file mode 100644 index 0000000000000000000000000000000000000000..add0ac2f18d3dd95dc1c087211637474cec89c7d --- /dev/null +++ b/benchmark/submissions/fat_former.json @@ -0,0 +1,22 @@ +{ + "method": "FatFormer", + "track": "forensynths", + "modality": "image", + "model_type": "fat_former", + "paper": "Liu et al., CVPR'24", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "official-weights", + "metrics": { + "acc": 98.39, + "map": 99.7, + "ap": null, + "auroc": null + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths-8 subset; official released weights", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/npr.json b/benchmark/submissions/npr.json new file mode 100644 index 0000000000000000000000000000000000000000..afcad59e4096fd78feae20758e297d8da0579714 --- /dev/null +++ b/benchmark/submissions/npr.json @@ -0,0 +1,22 @@ +{ + "method": "NPR", + "track": "forensynths", + "modality": "image", + "model_type": "npr", + "paper": "Tan et al., CVPR'24", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "official-weights", + "metrics": { + "acc": 91.83, + "map": 87.09, + "ap": null, + "auroc": null + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths-8; official ProGAN weights", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/restrav.json b/benchmark/submissions/restrav.json new file mode 100644 index 0000000000000000000000000000000000000000..8366417b79eb03c5e7205f06920886eaa35e3367 --- /dev/null +++ b/benchmark/submissions/restrav.json @@ -0,0 +1,22 @@ +{ + "method": "ReStraV", + "track": "genvideo-val", + "modality": "video", + "model_type": "restrav", + "paper": "2025", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "frozen-probe", + "metrics": { + "acc": null, + "map": null, + "ap": 96.67, + "auroc": 97.06 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "Frozen DINOv2 trajectory geometry + MLP", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/safe.json b/benchmark/submissions/safe.json new file mode 100644 index 0000000000000000000000000000000000000000..5fc07feaa7b1c9932a9055a91b243b81b0009f50 --- /dev/null +++ b/benchmark/submissions/safe.json @@ -0,0 +1,22 @@ +{ + "method": "SAFE", + "track": "forensynths", + "modality": "image", + "model_type": "safe", + "paper": "Li et al., KDD'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "from-scratch", + "metrics": { + "acc": null, + "map": 94.13, + "ap": null, + "auroc": 93.68 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "ForenSynths; trained from scratch", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/spai_official.json b/benchmark/submissions/spai_official.json new file mode 100644 index 0000000000000000000000000000000000000000..94b89860a136fef5adc45767b99890c3824d87db --- /dev/null +++ b/benchmark/submissions/spai_official.json @@ -0,0 +1,22 @@ +{ + "method": "SPAI (official weights)", + "track": "genimage-sdv1.4", + "modality": "image", + "model_type": "spai", + "paper": "Karageorgiou et al., CVPR'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "official-weights", + "metrics": { + "acc": null, + "map": null, + "ap": 94.01, + "auroc": 98.45 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "GenImage SD-v1.4; vendored official architecture, fixed-res 2x2x224", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/spai_scratch.json b/benchmark/submissions/spai_scratch.json new file mode 100644 index 0000000000000000000000000000000000000000..5b990b2d2ee34527ff16a7a07161f2079bec0d8c --- /dev/null +++ b/benchmark/submissions/spai_scratch.json @@ -0,0 +1,22 @@ +{ + "method": "SPAI (from scratch)", + "track": "genimage-sdv1.4", + "modality": "image", + "model_type": "spai", + "paper": "Karageorgiou et al., CVPR'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "from-scratch", + "metrics": { + "acc": null, + "map": null, + "ap": null, + "auroc": 97.66 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "DMID Latent-Diffusion vs COCO; SynthScope fixed-resolution variant", + "synthscope_version": "0.1.0" +} diff --git a/benchmark/submissions/vib_net.json b/benchmark/submissions/vib_net.json new file mode 100644 index 0000000000000000000000000000000000000000..0a4ec342ce81bdbd3f54f9274337171737c9f475 --- /dev/null +++ b/benchmark/submissions/vib_net.json @@ -0,0 +1,22 @@ +{ + "method": "VIB-Net", + "track": "genimage-sdv1.4", + "modality": "image", + "model_type": "vib_net", + "paper": "Zhang et al., CVPR'25", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "from-scratch", + "metrics": { + "acc": null, + "map": 99.33, + "ap": null, + "auroc": 99.33 + }, + "per_generator": null, + "submitted_by": "SynthScope", + "date": "2026-06-02", + "verified": true, + "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", + "protocol_notes": "GenImage SD-v1.4 single real/fake pool (mAP == AUROC by construction)", + "synthscope_version": "0.1.0" +} diff --git a/conf/config.yaml b/conf/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..147ee54b7a3389aa21c6d0b6e0a945c13d83235d --- /dev/null +++ b/conf/config.yaml @@ -0,0 +1,21 @@ +defaults: + - trainer@TrainerConfig: default + - model@ModelConfig: default + - dataset@DatasetConfig: image/genimage + - _self_ + +hydra: + job: + chdir: false + +metrics: + - real_accuracy + - fake_accuracy + +logger: + mode: offline + type: wandb + project: SynthScope + name: SynthScope + tags: + - SynthScope diff --git a/conf/dataset/image/aigcdetect.yaml b/conf/dataset/image/aigcdetect.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2cc84facb87cee3f588453240a9d8efb5d56046 --- /dev/null +++ b/conf/dataset/image/aigcdetect.yaml @@ -0,0 +1,14 @@ +name: aigcdetect +# AIGCDetectBenchmark 16-generator test set (ModelScope aemilia/AIGCDetectionBenchmark). +# Used by Co-Spy (ProGAN/AIGCDetect table) and AIDE. +root_dir: /hddstore/ssalehi/SynthScope/AIGCDetectBenchmark +test_batch_size: 512 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/aigibench.yaml b/conf/dataset/image/aigibench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acde566353bb7a46cc42885d6cfbbc1f0dc156dc --- /dev/null +++ b/conf/dataset/image/aigibench.yaml @@ -0,0 +1,23 @@ +name: aigibench +setting: Setting-II +root_dir: /hddstore/ssalehi/SynthScope/AIGIBench +train_batch_size: 512 +val_batch_size: 2048 +test_batch_size: 2048 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +train_transform_list: + - resize_short_256 + - gaussian_blur + - jpeg_compression + - random_crop_224 + - random_horizontal_flip + - to_dtype + - normalize_clip +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/cnndetection.yaml b/conf/dataset/image/cnndetection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..523adbcbf72b767ee2f9dbbe13775d3748fe0079 --- /dev/null +++ b/conf/dataset/image/cnndetection.yaml @@ -0,0 +1,23 @@ +name: cnndetection +root_dir: /hddstore/ssalehi/SynthScope/CNNDetection +source_url: https://drive.google.com/drive/u/0/folders/1RwCSaraEUctIwFgoQXWMKFvW07gM80_3 +train_batch_size: 512 +val_batch_size: 2048 +test_batch_size: 2048 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +train_transform_list: + - resize_short_256 + - gaussian_blur + - jpeg_compression + - random_crop_224 + - random_horizontal_flip + - to_dtype + - normalize_clip +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/commgen15.yaml b/conf/dataset/image/commgen15.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b92ab420158c3d3685202bac8fd42a396929655 --- /dev/null +++ b/conf/dataset/image/commgen15.yaml @@ -0,0 +1,15 @@ +name: commgen15 +# CommGen15 — 15 commercial-generator test set introduced by PGC (Table 3). +# (HF xiaoyuzhou68/CommGen15) +root_dir: /hddstore/ssalehi/SynthScope/CommGen15 +hf_repo_id: xiaoyuzhou68/CommGen15 +test_batch_size: 512 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/cospy.yaml b/conf/dataset/image/cospy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be9b2f70082967437f2d2d362365999cdfcfabde --- /dev/null +++ b/conf/dataset/image/cospy.yaml @@ -0,0 +1,51 @@ +name: cospy +root_dir: /hddstore/ssalehi/SynthScope/CoSpyBench + +repo_id: ruojiruoli/Co-Spy-Bench +real_repo_id: ruojiruoli/Co-Spy-Misc +real_filename: real_image_examples.zip +include_real: true + +source_datasets: + - mscoco + - flickr + - cc3m + - textcaps + - sbu + +generator_models: + - CompVis@ldm-text2im-large-256 + - CompVis@stable-diffusion-v1-4 + - runwayml@stable-diffusion-v1-5 + - segmind@SSD-1B + - segmind@tiny-sd + - segmind@SegMoE-SD-4x2-v0 + - segmind@small-sd + - stabilityai@stable-diffusion-2-1 + - stabilityai@stable-diffusion-3-medium-diffusers + - stabilityai@sdxl-turbo + - stabilityai@stable-diffusion-2 + - stabilityai@stable-diffusion-xl-base-1.0 + - playgroundai@playground-v2.5-1024px-aesthetic + - playgroundai@playground-v2-1024px-aesthetic + - playgroundai@playground-v2-512px-base + - playgroundai@playground-v2-256px-base + - PixArt-alpha@PixArt-XL-2-1024-MS + - PixArt-alpha@PixArt-XL-2-512x512 + - latent-consistency@lcm-lora-sdxl + - latent-consistency@lcm-lora-sdv1-5 + - black-forest-labs@FLUX.1-schnell + - black-forest-labs@FLUX.1-dev + +train_batch_size: 16 +val_batch_size: 16 +test_batch_size: 16 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +train_transform_list: [] +eval_transform_list: + - resize_short_384 + - center_crop_384 + - to_dtype diff --git a/conf/dataset/image/cospy_subset.yaml b/conf/dataset/image/cospy_subset.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5103054ce6933277dd495e46b6a3b2c0933ea096 --- /dev/null +++ b/conf/dataset/image/cospy_subset.yaml @@ -0,0 +1,23 @@ +# CO-SPY from-scratch: balanced ~120K ProGAN subset for train; full ForenSynths val/test. +# 'name' must be a registered dataset module -> reuse cnndetection (same gen/{0_real,1_fake} layout). +name: cnndetection +root_dir: /hddstore/ssalehi/SynthScope/spai_train/cospy_subset +train_batch_size: 32 +val_batch_size: 64 +test_batch_size: 64 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: false +# CO-SPY normalizes per-branch INSIDE forward; feed raw [0,1] (to_dtype scales uint8->[0,1]). +train_transform_list: + - resize_short_256 + - gaussian_blur + - jpeg_compression + - random_crop_224 + - random_horizontal_flip + - to_dtype +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype diff --git a/conf/dataset/image/diffusion1kstep.yaml b/conf/dataset/image/diffusion1kstep.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91bb00dbf7bbaf3f9e0b3eb9de8ad6bb66d7ab2f --- /dev/null +++ b/conf/dataset/image/diffusion1kstep.yaml @@ -0,0 +1,14 @@ +name: diffusion1kstep +# Diffusion-1kStep self-synthesis test set (DALLE, ddpm, guided-diffusion, improved-diffusion, +# midjourney). Used by NPR (Table 5). +root_dir: /hddstore/ssalehi/SynthScope/Diffusion1kStep +test_batch_size: 512 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/diffusionforensics.yaml b/conf/dataset/image/diffusionforensics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50864d108b145253d85fa8f0b23ae347b79ccb12 --- /dev/null +++ b/conf/dataset/image/diffusionforensics.yaml @@ -0,0 +1,14 @@ +name: diffusionforensics +# DiffusionForensics / DIRE diffusion test set (8 subsets: adm, ddpm, iddpm, ldm, pndm, sdv1, +# sdv2, vqdiffusion). Used by NPR (Table 3) and FatFormer's diffusion table. +root_dir: /hddstore/ssalehi/SynthScope/DiffusionForensics +test_batch_size: 512 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/ditfake.yaml b/conf/dataset/image/ditfake.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b98b71bfdf97e0ffa50a8a4f8585eb0f173d19f --- /dev/null +++ b/conf/dataset/image/ditfake.yaml @@ -0,0 +1,13 @@ +name: ditfake +root_dir: /hddstore/ssalehi/SynthScope/DiTFake +repo_id: lioooox/DiTFake +test_batch_size: 2048 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/freqnetdetection.yaml b/conf/dataset/image/freqnetdetection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00d7adb6061b88b43eaae4b30310e6ebfe5b05ff --- /dev/null +++ b/conf/dataset/image/freqnetdetection.yaml @@ -0,0 +1,23 @@ +name: freqnetdetection +root_dir: /nvmestore/ssalehi/SynthScope/freqnet +source_url: https://drive.google.com/drive/folders/11E0Knf9J1qlv2UuTnJSOFUjIIi90czSj +train_batch_size: 512 +val_batch_size: 2048 +test_batch_size: 2048 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +train_transform_list: + - resize_short_256 + - gaussian_blur + - jpeg_compression + - random_crop_224 + - random_horizontal_flip + - to_dtype + - normalize_clip +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/genimage.yaml b/conf/dataset/image/genimage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af0d728572687bffc78a0d019075703da84be9b4 --- /dev/null +++ b/conf/dataset/image/genimage.yaml @@ -0,0 +1,27 @@ +name: genimage +root_dir: /hddstore/ssalehi/SynthScope/GenImage +source_url: https://drive.google.com/drive/folders/1jGt10bwTbhEZuGXLyvrCuxOI0cBqQ1FS +# HuggingFace mirror (used instead of source_url when hf_repo_id is set). +# hf_subdir selects one GenImage generator dir; null/"" downloads the whole repo. +hf_repo_id: jzousz/GenImage +hf_subdir: stable_diffusion_v_1_4 +train_batch_size: 512 +val_batch_size: 2048 +test_batch_size: 2048 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +train_transform_list: + - resize_short_256 + - gaussian_blur + - jpeg_compression + - random_crop_224 + - random_horizontal_flip + - to_dtype + - normalize_clip +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/genimage_full.yaml b/conf/dataset/image/genimage_full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08aa007b7b80473661aa3da2fb8a77cbd7dfc922 --- /dev/null +++ b/conf/dataset/image/genimage_full.yaml @@ -0,0 +1,15 @@ +name: genimage_full +# Full 8-generator GenImage cross-generator test set (Midjourney, SD-v1.4/1.5, ADM, GLIDE, +# Wukong, VQDM, BigGAN). The single-pool SD-v1.4 slice is the separate `genimage` dataset. +root_dir: /hddstore/ssalehi/SynthScope/GenImage_full +hf_repo_id: jzousz/GenImage +test_batch_size: 512 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/spai_ldm.yaml b/conf/dataset/image/spai_ldm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32b831fd481a1a22eef57184f1e3097dffe371dc --- /dev/null +++ b/conf/dataset/image/spai_ldm.yaml @@ -0,0 +1,22 @@ +# 'name' must be a key in dataset.py SynthScopeModuleDict; the staged data uses the +# CNNDetection gen/{0_real,1_fake} ImageFolder layout, so reuse that module. +name: cnndetection +root_dir: /hddstore/ssalehi/SynthScope/spai_train/staged +train_batch_size: 32 +val_batch_size: 64 +test_batch_size: 64 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: false +# SPAI expects images in [0,1]; ImageNet normalization happens INSIDE the model. +# So NO normalize_* transform here (to_dtype scales uint8->[0,1]). +train_transform_list: + - resize_short_256 + - random_crop_224 + - random_horizontal_flip + - to_dtype +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype diff --git a/conf/dataset/image/synthbuster.yaml b/conf/dataset/image/synthbuster.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46823cb760e59554a55a513930523bf5c354442f --- /dev/null +++ b/conf/dataset/image/synthbuster.yaml @@ -0,0 +1,11 @@ +name: synthbuster +root_dir: /hddstore/ssalehi/SynthScope/Synthbuster +zenodo_record: "10066460" +test_batch_size: 32 +num_workers: 8 +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/image/universialfakedetection.yaml b/conf/dataset/image/universialfakedetection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c58f0cc1666df6e07be085291303f092c90f0a5 --- /dev/null +++ b/conf/dataset/image/universialfakedetection.yaml @@ -0,0 +1,23 @@ +name: universialfakedetection +root_dir: /nvmestore/ssalehi/SynthScope/universialfakedetection +source_url: https://drive.google.com/file/d/1FXlGIRh_Ud3cScMgSVDbEWmPDmjcrm1t/view +train_batch_size: 512 +val_batch_size: 2048 +test_batch_size: 2048 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +train_transform_list: + - resize_short_256 + - gaussian_blur + - jpeg_compression + - random_crop_224 + - random_horizontal_flip + - to_dtype + - normalize_clip +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/dataset/video/comgenvid.yaml b/conf/dataset/video/comgenvid.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a859853ccc241063a6011ce6cf4f44cbf7f9fb2a --- /dev/null +++ b/conf/dataset/video/comgenvid.yaml @@ -0,0 +1,17 @@ +name: comgenvid +# ComGenVid (STALL's new benchmark) — Sora + VEO3 fakes vs MSVD real. Used by STALL. +# Lay out as /test//{0_real,1_fake}/*.mp4. The HF repo OmerXYZ/comgenvid also +# ships precomputed embeddings for STALL's official embedding-based eval. +root_dir: /hddstore/ssalehi/SynthScope/ComGenVid +hf_repo_id: OmerXYZ/comgenvid +num_frames: 16 +fps: 8 +window_seconds: 2.0 +frame_size: 224 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +eval_transform_list: + - normalize_dinov3 diff --git a/conf/dataset/video/evalcrafter.yaml b/conf/dataset/video/evalcrafter.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f431d4648dec0574307290afd1ea5fc8a61cd9c0 --- /dev/null +++ b/conf/dataset/video/evalcrafter.yaml @@ -0,0 +1,24 @@ +name: evalcrafter +# EvalCrafter benchmark (Liu et al. 2024), used by D3. Lay out videos as +# ///{0_real,1_fake}/*.mp4. +root_dir: /hddstore/ssalehi/SynthScope/EvalCrafter +# EvalCrafterDataset downloads the per-generator *.tar.gz from this repo and extracts each into +# /test/1_fake// (all clips are AI-generated — no reals). +hf_repo_id: RaphaelLiu/EvalCrafter_T2V_Dataset +num_frames: 16 +fps: 8 +window_seconds: 2.0 +frame_size: 224 +train_batch_size: 8 +val_batch_size: 8 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +# Normalization happens HERE (not in the model). normalize_clip matches D3's default +# XCLIP/CLIP backbone; switch to normalize_dinov2 for a DINOv2/ResNet backbone. +train_transform_list: + - normalize_clip +eval_transform_list: + - normalize_clip diff --git a/conf/dataset/video/genvidbench.yaml b/conf/dataset/video/genvidbench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18f3735fcdc13c7bbc3a981de58592f4a1b600c4 --- /dev/null +++ b/conf/dataset/video/genvidbench.yaml @@ -0,0 +1,24 @@ +name: genvidbench +# GenVidBench benchmark (Ni et al. 2025), used by ReStraV. Lay out videos as +# ///{0_real,1_fake}/*.mp4. +root_dir: /hddstore/ssalehi/SynthScope/GenVidBench +# GenVidBenchDataset downloads the per-source rar/7z archives from this repo and extracts reals +# (Vript, HD-VG-130M) -> test/0_real/ and the 8 generators -> test/1_fake/ (needs the `unrar` binary). +hf_repo_id: bitmind/GenVidBench +num_frames: 24 +fps: null +window_seconds: 2.0 +frame_size: 224 +train_batch_size: 8 +val_batch_size: 8 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +# Normalization happens HERE (not in the model). normalize_dinov2 = ImageNet stats, what +# DINOv2 (ReStraV's backbone) expects. +train_transform_list: + - normalize_dinov2 +eval_transform_list: + - normalize_dinov2 diff --git a/conf/dataset/video/genvideo.yaml b/conf/dataset/video/genvideo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..219185042463d1041eea68ef358b5e702344841e --- /dev/null +++ b/conf/dataset/video/genvideo.yaml @@ -0,0 +1,25 @@ +name: genvideo +# GenVideo benchmark (DeMamba, Chen et al. 2024). Lay out videos as +# ///{0_real,1_fake}/*.mp4 (test split is what D3 uses). +root_dir: /hddstore/ssalehi/SynthScope/GenVideo +# Set to a Hugging Face dataset repo to enable the download stub; null = manual placement. +hf_repo_id: null +# Frame sampling: D3 uses ~8 fps over up to 2 s. +num_frames: 16 +fps: 8 +window_seconds: 2.0 +frame_size: 224 +train_batch_size: 8 +val_batch_size: 8 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +# Frames are decoded to 224x224 in [0, 1]; normalization happens HERE (not in the model). +# normalize_clip matches D3's default XCLIP/CLIP backbone; switch to normalize_dinov2 for a +# DINOv2/ResNet backbone. +train_transform_list: + - normalize_clip +eval_transform_list: + - normalize_clip diff --git a/conf/dataset/video/physics_iq.yaml b/conf/dataset/video/physics_iq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..505ad95376fa6c98f8d4d9eeaf397eeecb9a57bb --- /dev/null +++ b/conf/dataset/video/physics_iq.yaml @@ -0,0 +1,24 @@ +name: physics_iq +# Physics-IQ benchmark (Motamed et al. 2025), used by ReStraV. Matched real/generated +# pairs. Lay out videos as ///{0_real,1_fake}/*.mp4. +root_dir: /hddstore/ssalehi/SynthScope/PhysicsIQ +# google-deepmind/physics-IQ-benchmark hosts the data; set this to enable the download +# stub once the desired split layout is known. null = manual placement. +hf_repo_id: null +num_frames: 24 +fps: null +window_seconds: 2.0 +frame_size: 224 +train_batch_size: 8 +val_batch_size: 8 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +# Normalization happens HERE (not in the model). normalize_dinov2 = ImageNet stats, what +# DINOv2 (ReStraV's backbone) expects. +train_transform_list: + - normalize_dinov2 +eval_transform_list: + - normalize_dinov2 diff --git a/conf/dataset/video/videofeedback.yaml b/conf/dataset/video/videofeedback.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71878a51b41ce98b84afdbf32c7dbbcb7afde27d --- /dev/null +++ b/conf/dataset/video/videofeedback.yaml @@ -0,0 +1,16 @@ +name: videofeedback +# VideoFeedback (TIGER-Lab) — 11 generators vs DiDeMo+Panda70M real. Used by STALL. +# Lay out as /test//{0_real,1_fake}/*.mp4. +root_dir: /hddstore/ssalehi/SynthScope/VideoFeedback +hf_repo_id: TIGER-Lab/VideoFeedback +num_frames: 16 +fps: 8 +window_seconds: 2.0 +frame_size: 224 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +eval_transform_list: + - normalize_dinov3 diff --git a/conf/dataset/video/videophy.yaml b/conf/dataset/video/videophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66e6e01f71b3707b9ae74190eb0bb39e2b44e31f --- /dev/null +++ b/conf/dataset/video/videophy.yaml @@ -0,0 +1,23 @@ +name: videophy +# VideoPhy benchmark (Bansal et al. 2024), used by D3. Lay out videos as +# ///{0_real,1_fake}/*.mp4. +root_dir: /hddstore/ssalehi/SynthScope/VideoPhy +# Set to a Hugging Face dataset repo to enable the download stub; null = manual placement. +hf_repo_id: null +num_frames: 16 +fps: 8 +window_seconds: 2.0 +frame_size: 224 +train_batch_size: 8 +val_batch_size: 8 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +# Normalization happens HERE (not in the model). normalize_clip matches D3's default +# XCLIP/CLIP backbone; switch to normalize_dinov2 for a DINOv2/ResNet backbone. +train_transform_list: + - normalize_clip +eval_transform_list: + - normalize_clip diff --git a/conf/dataset/video/vidprom.yaml b/conf/dataset/video/vidprom.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01a75c16053d40a8d47830294336c5c873ea8390 --- /dev/null +++ b/conf/dataset/video/vidprom.yaml @@ -0,0 +1,26 @@ +name: vidprom +# VidProM benchmark (Wang & Yang 2024). Shared by D3 and ReStraV. Lay out videos as +# ///{0_real,1_fake}/*.mp4. +root_dir: /hddstore/ssalehi/SynthScope/VidProM +# WenhaoWang/VidProM on the Hugging Face Hub hosts the prompts/videos; set this to enable +# the download stub once the desired split layout is known. null = manual placement. +hf_repo_id: null +# 24 frames evenly across 2 s works for both ReStraV (needs >=8) and D3 (needs >=4). +# Override num_frames/fps on the command line if you want D3's 16-frame@8fps sampling. +num_frames: 24 +fps: null +window_seconds: 2.0 +frame_size: 224 +train_batch_size: 8 +val_batch_size: 8 +test_batch_size: 8 +num_workers: 4 +persistent_workers: true +pin_memory: true +prefetch: false +# Normalization happens HERE (not in the model). Default normalize_dinov2 = ReStraV's +# DINOv2 backbone; for D3 (XCLIP/CLIP) override to normalize_clip in the experiment script. +train_transform_list: + - normalize_dinov2 +eval_transform_list: + - normalize_dinov2 diff --git a/conf/model/default.yaml b/conf/model/default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6914efcbf60bd47eebc03b59ae0971d47635a61d --- /dev/null +++ b/conf/model/default.yaml @@ -0,0 +1,5 @@ +model_type: clip +backbone_name: CLIP:ViT-L/14 +bottleneck_dim: 64 +use_adapter: false +freeze_backbone: true diff --git a/conf/model/image/aide.yaml b/conf/model/image/aide.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0e5233dd7a66b5c7257e4bbd942b9d90d267295 --- /dev/null +++ b/conf/model/image/aide.yaml @@ -0,0 +1,21 @@ +defaults: + - /model/default + - _self_ + +model_type: aide + +convnext_model_name: convnext_xxlarge +convnext_pretrained: laion2b_s34b_b82k_augreg_soup +freeze_convnext: true +semantic_dim: 3072 +semantic_proj_dim: 256 + +resnet_pretrained: true +head_hidden: 1024 +num_classes: 2 + +dct: + window_size: 32 + stride: 16 + output_size: 256 + grade_N: 6 diff --git a/conf/model/image/co_spy.yaml b/conf/model/image/co_spy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac06a255f2baf813e4d60cb74436e743a33b1f5f --- /dev/null +++ b/conf/model/image/co_spy.yaml @@ -0,0 +1,21 @@ +defaults: + - /model/default + - _self_ + +model_type: co_spy + +semantic_model_name: ViT-SO400M-14-SigLIP-384 +semantic_pretrained: webli +semantic_image_size: 384 + +vae_id: CompVis/stable-diffusion-v1-4 +vae_subfolder: vae +artifact_image_size: 224 +artifact_residual_scale: 14.285714285714286 + +coeff_dropout: 0.3 +feat_interp_prob: 0.5 +feat_interp_alpha: 0.2 + +num_classes: 2 +freeze_backbone: true diff --git a/conf/model/image/cure.yaml b/conf/model/image/cure.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08c15e3b588665bff22e50c3bbcf3562e664f12b --- /dev/null +++ b/conf/model/image/cure.yaml @@ -0,0 +1,15 @@ +defaults: + - /model/default + - _self_ + +# CuRe (arXiv 2609.14316): frozen Perception Encoder Core L/14 @336 + Q/V LoRA + source-response +# subspace + linear head. Official released adapter loads key-for-key as cure.pth (25 MB); the +# frozen base (facebook/PE-Core-L14-336, Apache-2.0, 2.7 GB) is fetched from the HF Hub on first use. +model_type: cure +backbone_name: PE-Core-L14-336 +freeze_backbone: true +base_repo_id: facebook/PE-Core-L14-336 +base_filename: PE-Core-L14-336.pt +base_checkpoint: null # local PE-Core-L14-336.pt override (or env SYNTHSCOPE_PE_CORE_CKPT) +load_base_backbone: true # false = build the architecture only (smoke tests / key checks) +label_smoothing: 0.0 diff --git a/conf/model/image/fat_former.yaml b/conf/model/image/fat_former.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60f632f7cb2ebed247718fb00e11bf7281f644f9 --- /dev/null +++ b/conf/model/image/fat_former.yaml @@ -0,0 +1,13 @@ +defaults: + - /model/default + - _self_ + +model_type: fat_former +backbone_name: CLIP:ViT-L/14 +num_vit_adapter: 8 +num_context_embedding: 8 +init_context_embedding: "" +num_classes: 2 +classnames: [real, fake] +freeze_backbone: true +force_fp32: true diff --git a/conf/model/image/npr.yaml b/conf/model/image/npr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8be41d60769934bd8788ea6276c5b783f2919dfa --- /dev/null +++ b/conf/model/image/npr.yaml @@ -0,0 +1,7 @@ +defaults: + - /model/default + - _self_ + +model_type: npr +num_classes: 2 +freeze_backbone: false diff --git a/conf/model/image/pgc.yaml b/conf/model/image/pgc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7988e6d9c008970a0d80e36761fddc85f3ef6eab --- /dev/null +++ b/conf/model/image/pgc.yaml @@ -0,0 +1,21 @@ +defaults: + - /model/default + - _self_ + +model_type: pgc +# RGB backbone: a Hugging Face DINO ViT (auto-downloaded). Paper default: DINOv2-Large (D=1024). +backbone_name: facebook/dinov2-large +freeze_backbone: true # backbone frozen; only the injected LoRA adapters train +# LoRA (paper defaults). The official released checkpoints carry LoRA on the MLP only (their attn +# target names did not match the HF DINOv2 module names), so to reproduce their exact recipe we target +# MLP only — a strictly smaller, better-conditioned adapter than tuning attention+MLP. +lora_rank: 8 +lora_alpha: 1.0 +lora_dropout: 0.1 +lora_targets: [mlp.fc1, mlp.fc2] +# Label smoothing (official: 0.1). PGC's compute_loss applies it directly (the trainer's CE is bypassed). +label_smoothing: 0.1 +# Peak-aggregation temperatures + RGB-branch weight (paper Eq. 4). +tau_rgb: 0.5 +tau_res: 0.5 +num_classes: 1 diff --git a/conf/model/image/safe.yaml b/conf/model/image/safe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcd9577b14469bde9e80305f59b75c8d3284d84b --- /dev/null +++ b/conf/model/image/safe.yaml @@ -0,0 +1,8 @@ +defaults: + - /model/default + - _self_ + +# SAFE builds its own lightweight ResNet-50-style trunk + DWT high-frequency front end; +# it does NOT use backbone_name. model_type=safe is dispatched before the CLIP fallback. +model_type: safe +freeze_backbone: false diff --git a/conf/model/image/spai.yaml b/conf/model/image/spai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa73ea5a11854a406e836f7adf8c3c085e1f4e40 --- /dev/null +++ b/conf/model/image/spai.yaml @@ -0,0 +1,18 @@ +defaults: + - /model/default + - _self_ + +model_type: spai +backbone_name: ViT-B/16 +freeze_backbone: true +input_size: 224 # per-patch size (the circular spectral mask is built at this size) +mask_radius: 16 +projection_dim: 1024 +# Spectral Context Attention over a grid of 224 patches (official fixed-res variant = 448 -> 2x2 grid). +# Feed the model 448x448 inputs (see the spai transform); set use_sca=false to recover the old +# single-patch fixed-resolution model. +use_sca: true +patch_size: 224 +patch_stride: 224 +num_heads: 12 +attn_embed_dim: 1536 \ No newline at end of file diff --git a/conf/model/image/vib_net.yaml b/conf/model/image/vib_net.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc373daf1cc1284d9f656f74b024b0b30069a6c --- /dev/null +++ b/conf/model/image/vib_net.yaml @@ -0,0 +1,15 @@ +defaults: + - /model/default + - _self_ + +model_type: vib_net +# Backbone: CLIP:ViT-L/14 is the shipped/working config (the released vib_net.pth is CLIP-based, GenImage +# mAP 83.26). A FLAVA-full path (the paper's Fig. 5 headline backbone) is also implemented in +# SynthScopeVIBNetModel, but frozen-FLAVA + VIB-head training was unstable here (val loss stayed at chance over +# 5 epochs), so CLIP is the default; set backbone_name: FLAVA:facebook/flava-full to experiment. +backbone_name: CLIP:ViT-L/14 +vib_hidden_dim: 1024 +vib_latent_dim: 256 +dropout: 0.5 +std_shift: 5.0 +freeze_backbone: true diff --git a/conf/model/video/d3.yaml b/conf/model/video/d3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0747df76a32943904588a33e118316cfc4b2161 --- /dev/null +++ b/conf/model/video/d3.yaml @@ -0,0 +1,16 @@ +defaults: + - /model/default + - _self_ + +model_type: d3 +# Frozen per-frame visual encoder (HF, matches official D3): clip | clip-32 | dinov2-base | dinov2-large | resnet18 +backbone_name: clip +# First-order inter-frame feature: l2 (paper default) | cos +first_order: l2 +# Decision threshold tau for argmax accuracy (AP/AUROC are threshold-free and primary). +threshold: 0.0 +# Sampling interval scale; a constant that does not change the std-based ranking. +dt: 1.0 +# Number of sampled frames per clip (must match the dataset config). +num_frames: 16 +freeze_backbone: true diff --git a/conf/model/video/restrav.yaml b/conf/model/video/restrav.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22693d2002aa381da0b2a2aaa9161c3e63151540 --- /dev/null +++ b/conf/model/video/restrav.yaml @@ -0,0 +1,12 @@ +defaults: + - /model/default + - _self_ + +model_type: restrav +# Frozen self-supervised backbone (torch.hub facebookresearch/dinov2). +backbone_name: dinov2_vits14 +# Number of sampled frames per clip (must match the dataset config). +num_frames: 24 +# Trainable classifier MLP hidden sizes (paper: two layers 64 -> 32). +mlp_hidden: [64, 32] +freeze_backbone: true diff --git a/conf/model/video/stall.yaml b/conf/model/video/stall.yaml new file mode 100644 index 0000000000000000000000000000000000000000..878a36f1fb792c565da7e4804e37f3e574786a44 --- /dev/null +++ b/conf/model/video/stall.yaml @@ -0,0 +1,27 @@ +defaults: + - /model/default + - _self_ + +model_type: stall +# Frozen DINOv3 ViT-L/16 — the backbone the shipped VATEX calibration params were computed with. +backbone_name: dinov3_vitl16 +freeze_backbone: true +# Per-video aggregation (paper): spatial = max over frames, temporal = min over frame-pairs. +spat_agg: max +temp_agg: min +# Frames sampled per clip (must match the dataset config). Paper uses 16. +num_frames: 16 +# Calibration / whitening params (real-data statistics). null => the VATEX/DINOv3 npz shipped +# next to the model. The model recomputes the params from real videos when recompute_params=true +# OR when this path does not exist, then saves them here. +stall_params_path: null +recompute_params: false +# When recomputing: which split of DatasetConfig to calibrate on, whether to keep only real +# (label 0) clips, and an optional cap on the number of clips (0 = all). Point DatasetConfig at +# a real-video set DISJOINT from your eval set to avoid leakage. +calib_split: train +calib_real_only: true +calib_max_videos: 0 +# DINOv3 backbone source (gated weights). null => DINO_V3_REPO_DIR / DINO_V3_WEIGHTS env vars. +dino_repo: null +dino_weights: null diff --git a/conf/trainer/default.yaml b/conf/trainer/default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c46b174b5905a0c31bc9cd75e91f44415cf9b3aa --- /dev/null +++ b/conf/trainer/default.yaml @@ -0,0 +1,13 @@ +epochs: 10 +lr: 0.0001 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +save_path: /nvmestore/ssalehi/SynthScope/models +seed: 42 +warmup: + mode: off + warmup_ratio: 0.05 + lr_scheduler: linear \ No newline at end of file diff --git a/conf/trainer/image/aide.yaml b/conf/trainer/image/aide.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bb7bfb1844b0822bc78514fd37bb68f8aacf8ba --- /dev/null +++ b/conf/trainer/image/aide.yaml @@ -0,0 +1,18 @@ +defaults: + - /trainer/default + - _self_ + +trainer_type: aide +epochs: 5 +lr: 1.0e-4 +weight_decay: 0.0 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: off + warmup_ratio: 0.05 + lr_scheduler: linear diff --git a/conf/trainer/image/co_spy.yaml b/conf/trainer/image/co_spy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ea2295dcdfc2235a906db4891349012fd70c6c7 --- /dev/null +++ b/conf/trainer/image/co_spy.yaml @@ -0,0 +1,18 @@ +defaults: + - /trainer/default + - _self_ + +trainer_type: co_spy +epochs: 10 +lr: 0.0001 +weight_decay: 0.0 +label_smoothing: 0.1 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: on + warmup_ratio: 0.05 + lr_scheduler: cosine diff --git a/conf/trainer/image/fat_former.yaml b/conf/trainer/image/fat_former.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b2529cf46d564de08a833ef5b2248f35da398cc --- /dev/null +++ b/conf/trainer/image/fat_former.yaml @@ -0,0 +1,18 @@ +defaults: + - /trainer/default + - _self_ + +trainer_type: fat_former +epochs: 25 +lr: 0.0001 +weight_decay: 0.0001 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: on + warmup_ratio: 0.05 + lr_scheduler: cosine diff --git a/conf/trainer/image/npr.yaml b/conf/trainer/image/npr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9da8b2d39204705876b095ed89e60c0dea6b38f5 --- /dev/null +++ b/conf/trainer/image/npr.yaml @@ -0,0 +1,18 @@ +defaults: + - /trainer/default + - _self_ + +trainer_type: npr +epochs: 10 +lr: 2.0e-4 +weight_decay: 0.0 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: off + warmup_ratio: 0.05 + lr_scheduler: linear diff --git a/conf/trainer/image/pgc.yaml b/conf/trainer/image/pgc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27ce08b698c59299c05a8ce8ad24dd695cb3fa10 --- /dev/null +++ b/conf/trainer/image/pgc.yaml @@ -0,0 +1,21 @@ +defaults: + - /trainer/default + - _self_ + +# PGC trains LoRA adapters + the residual stream + PGCM + classifier (DINO backbone frozen). +# Official recipe (xiaoyu6868/PGC): AdamW, lr 5e-5, weight_decay 0.05, 100 epochs (--niter 100), +# CosineAnnealingLR over all steps, label smoothing 0.1, effective batch 128. +trainer_type: pgc +epochs: 100 +lr: 5.0e-5 +weight_decay: 0.05 +label_smoothing: 0.1 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: true +save_policy: max +warmup: + mode: on + warmup_ratio: 0.05 + lr_scheduler: cosine diff --git a/conf/trainer/image/safe.yaml b/conf/trainer/image/safe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3aab6496ee1fd1d21ba316ee0fc99f00124962d --- /dev/null +++ b/conf/trainer/image/safe.yaml @@ -0,0 +1,15 @@ +epochs: 10 +lr: 0.0001 +weight_decay: 0.05 +label_smoothing: 0.1 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +save_path: /nvmestore/ssalehi/SynthScope/models +seed: 42 +warmup: + mode: on + warmup_ratio: 0.05 + lr_scheduler: linear diff --git a/conf/trainer/image/spai.yaml b/conf/trainer/image/spai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e19948a913d425a0610b66448890d763d37d6412 --- /dev/null +++ b/conf/trainer/image/spai.yaml @@ -0,0 +1,18 @@ +defaults: + - /trainer/default + - _self_ + +trainer_type: spai +epochs: 35 +lr: 5.0e-4 +weight_decay: 0.05 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: on + warmup_ratio: 0.143 + lr_scheduler: cosine diff --git a/conf/trainer/image/vib_net.yaml b/conf/trainer/image/vib_net.yaml new file mode 100644 index 0000000000000000000000000000000000000000..846f51521757141caf499657e934f8064d442e61 --- /dev/null +++ b/conf/trainer/image/vib_net.yaml @@ -0,0 +1,18 @@ +defaults: + - /trainer/default + - _self_ + +trainer_type: vib_net +epochs: 10 +lr: 0.0001 +weight_decay: 0.05 +label_smoothing: 0.1 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: on + warmup_ratio: 0.05 + lr_scheduler: cosine \ No newline at end of file diff --git a/conf/trainer/video/d3.yaml b/conf/trainer/video/d3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5200366aa484fa1afe4d0b926f3ba11e827287e2 --- /dev/null +++ b/conf/trainer/video/d3.yaml @@ -0,0 +1,21 @@ +defaults: + - /trainer/default + - _self_ + +# D3 is training-free: it carries one dummy parameter so AdamW does not crash, but +# "training" has zero effect. One epoch is enough; the val/test passes produce the +# real (threshold-free AP/AUROC) result. +trainer_type: d3 +epochs: 1 +lr: 1.0e-6 +weight_decay: 0.0 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: off + warmup_ratio: 0.0 + lr_scheduler: linear diff --git a/conf/trainer/video/restrav.yaml b/conf/trainer/video/restrav.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77be221bc07e9d1cc6efe286673cb318db046601 --- /dev/null +++ b/conf/trainer/video/restrav.yaml @@ -0,0 +1,21 @@ +defaults: + - /trainer/default + - _self_ + +# ReStraV trains only the small MLP classifier on top of the frozen DINOv2 features, +# so a short run converges quickly. Official recipe: plain Adam (no weight decay), 10 epochs +# (the trainer uses AdamW, so weight_decay=0.0 makes it Adam-equivalent). +trainer_type: restrav +epochs: 10 +lr: 1.0e-3 +weight_decay: 0.0 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: off + warmup_ratio: 0.05 + lr_scheduler: cosine diff --git a/conf/trainer/video/stall.yaml b/conf/trainer/video/stall.yaml new file mode 100644 index 0000000000000000000000000000000000000000..209385df613ada286a6694ead25578b045a1de3c --- /dev/null +++ b/conf/trainer/video/stall.yaml @@ -0,0 +1,21 @@ +defaults: + - /trainer/default + - _self_ + +# STALL is training-free: it carries one dummy parameter so AdamW does not crash, but +# "training" has zero effect. One epoch is enough; the val/test passes produce the real +# (threshold-free AP/AUROC) result. +trainer_type: stall +epochs: 1 +lr: 1.0e-6 +weight_decay: 0.0 +label_smoothing: 0.0 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: false +save_policy: max +warmup: + mode: off + warmup_ratio: 0.0 + lr_scheduler: linear diff --git a/config_utils.py b/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dd09e255a4a12fa32b8fd4629982bf79cd14a654 --- /dev/null +++ b/config_utils.py @@ -0,0 +1,15 @@ +"""Load the Hydra-composed app config from non-Hydra entrypoints (scripts, tests).""" +from pathlib import Path + +from omegaconf import DictConfig + +_CONF_DIR = Path(__file__).resolve().parent / "conf" + + +def compose_config(overrides: list[str] | None = None) -> DictConfig: + """Compose ``conf/config.yaml`` with optional Hydra overrides.""" + from hydra import compose, initialize_config_dir + + overrides = overrides or [] + with initialize_config_dir(version_base=None, config_dir=str(_CONF_DIR)): + return compose(config_name="config", overrides=overrides) diff --git a/dataset.py b/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..4896d3513af38c2ed4d268a4fcd19ba9cfc0c099 --- /dev/null +++ b/dataset.py @@ -0,0 +1,2521 @@ +from torch.utils.data import Dataset +from typing import Literal +from pathlib import Path +import hashlib +import os +import re +import zipfile +import tarfile +import shutil +import subprocess +from matplotlib import pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from decord import VideoReader, cpu +from torch.utils.data import DataLoader +from transformations import AIGITransformations +from torchvision.io import decode_image, ImageReadMode +from PIL import Image +from config_utils import compose_config + + +def _decode_rgb(path) -> torch.Tensor: + """Decode an image to a uint8 [3, H, W] RGB tensor. + + Uses torchvision's fast native decoder, falling back to PIL for the rare files + its backend rejects (e.g. CMYK / progressive / slightly-malformed JPEGs that + raise "Internal error"). Keeps the uint8 CHW contract the transform pipeline expects. + """ + try: + return decode_image(str(path), mode=ImageReadMode.RGB) + except Exception: + pass + try: + img = Image.open(str(path)).convert("RGB") + return torch.from_numpy(np.array(img)).permute(2, 0, 1).contiguous() + except Exception: + # Corrupt / truncated / empty file (some GenImage PNGs are 0-byte): don't kill the + # whole run — return a black 256x256 image so this sample is just uninformative. + return torch.zeros(3, 256, 256, dtype=torch.uint8) + + +IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp") +VIDEO_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv", ".webm", ".m4v", ".gif") + +def _iter_images(directory: Path): + return (p for p in directory.rglob("*") if p.is_file() and p.suffix.lower() in IMAGE_EXTENSIONS) + + +def _has_images(directory: Path) -> bool: + return directory.is_dir() and any(_iter_images(directory)) + + +def _iter_videos(directory: Path): + return (p for p in directory.rglob("*") if p.is_file() and p.suffix.lower() in VIDEO_EXTENSIONS) + + +def _has_videos(directory: Path) -> bool: + return directory.is_dir() and any(_iter_videos(directory)) + + +def sample_video_frames( + path: str | Path, + num_frames: int, + fps: float | None = None, + window_seconds: float = 2.0, + size: int = 224, + window_center: bool = False, + interp: str = "bilinear", + edge_crop: float = 0.0, +) -> torch.Tensor: + """Decode a clip of exactly ``num_frames`` frames and return ``[T, 3, size, size]`` float in ``[0, 1]``. + + Frames are sampled over a ``window_seconds`` window. By default the window starts at the + beginning of the video; set ``window_center=True`` to center it at the video midpoint (this is + what the official ReStraV pipeline does). When ``fps`` is given (e.g. D3 samples at 8 fps), the + window is shortened to ``num_frames / fps`` seconds so the effective sampling rate matches + ``fps``; when ``fps`` is ``None`` the frames are spread evenly across ``window_seconds``. The + resize interpolation is ``interp`` (``"bilinear"`` default; ``"bicubic"`` for ReStraV, with + antialiasing, matching the official preprocessing). The output frame count is always + ``num_frames`` so the default collate can stack clips into ``[B, T, 3, size, size]``. + """ + try: + reader = VideoReader(str(path), ctx=cpu(0)) + num_source = len(reader) + except Exception: + return torch.zeros(num_frames, 3, size, size) + if num_source == 0: + return torch.zeros(num_frames, 3, size, size) + + avg_fps = float(reader.get_avg_fps()) or 30.0 + duration = num_source / avg_fps + window = min(window_seconds, duration) + if fps and fps > 0: + window = min(window, num_frames / float(fps)) + + n_window = max(int(round(window * avg_fps)), 1) # frames spanning the window + start_index = 0 + if window_center: # center the window at the midpoint + start_index = int(round((duration / 2.0 - window / 2.0) * avg_fps)) + start_index = min(max(start_index, 0), num_source - 1) + end_index = min(start_index + n_window - 1, num_source - 1) + indices = np.linspace(start_index, end_index, num=num_frames) + indices = np.clip(np.rint(indices).astype(np.int64), 0, num_source - 1) + + # decord's batched get_batch raises a size-mismatch DECORDError on videos whose frames + # vary in resolution (or are partly corrupt); fall back to per-frame decode+resize, and + # finally to a black clip, so one bad video never crashes the whole DataLoader worker. + def _resize(t): # t: [T,3,H,W] float + if edge_crop and edge_crop > 0.0: + # Crop the central (1 - 2*edge_crop) region off each frame BEFORE resizing — the official + # D3 preprocessing trims borders/letterboxing that otherwise dominate the frozen-feature + # trajectory. Default 0.0 leaves D3-without-crop / ReStraV / STALL untouched. + h, w = t.shape[-2], t.shape[-1] + ch, cw = int(round(h * edge_crop)), int(round(w * edge_crop)) + if h - 2 * ch >= 8 and w - 2 * cw >= 8: + t = t[..., ch:h - ch, cw:w - cw] + if t.shape[-2:] != (size, size): + # Default ("bilinear") preserves the original no-antialias behavior (D3/STALL unchanged); + # "bicubic" enables antialiasing to match the official ReStraV resize. + t = F.interpolate(t, size=(size, size), mode=interp, align_corners=False, + antialias=(interp == "bicubic")) + return t + + try: + frames = reader.get_batch(indices.tolist()).asnumpy() # [T, H, W, 3] uint8 + clip = torch.from_numpy(frames).permute(0, 3, 1, 2).float() / 255.0 + clip = _resize(clip) + except Exception: + per_frame = [] + for i in indices.tolist(): + try: + f = reader[i].asnumpy() # [H, W, 3] + ft = torch.from_numpy(f).permute(2, 0, 1).unsqueeze(0).float() / 255.0 + per_frame.append(_resize(ft)) + except Exception: + per_frame.append(torch.zeros(1, 3, size, size)) + clip = torch.cat(per_frame, dim=0) if per_frame else torch.zeros(num_frames, 3, size, size) + # Coerce to exactly 3 channels: some clips decode to 1 (grayscale), 4 (RGBA) or more + # (e.g. packed/stereo streams) channels, which breaks the 3-channel Normalize downstream. + c = clip.shape[1] + if c == 1: + clip = clip.repeat(1, 3, 1, 1) + elif c != 3: + clip = clip[:, :3] + return clip.clamp_(0.0, 1.0) + + +def _safe_extract_zip(zip_path: str | Path, destination_dir: str | Path) -> None: + """Extract a zip file without allowing path traversal outside destination_dir.""" + zip_path = Path(zip_path) + destination_dir = Path(destination_dir) + destination_dir.mkdir(parents=True, exist_ok=True) + destination_root = destination_dir.resolve() + + with zipfile.ZipFile(zip_path, "r") as zf: + for member in zf.infolist(): + member_path = (destination_dir / member.filename).resolve() + if not member_path.is_relative_to(destination_root): + raise RuntimeError(f"Refusing to extract unsafe zip member: {member.filename}") + zf.extractall(destination_dir) + + +def _safe_extract_tar(tar_path: str | Path, destination_dir: str | Path) -> None: + """Extract a tar file without allowing path traversal outside destination_dir.""" + tar_path = Path(tar_path) + destination_dir = Path(destination_dir) + destination_dir.mkdir(parents=True, exist_ok=True) + destination_root = destination_dir.resolve() + + with tarfile.open(tar_path, "r:*") as tf: + for member in tf.getmembers(): + member_path = (destination_dir / member.name).resolve() + if not member_path.is_relative_to(destination_root): + raise RuntimeError(f"Refusing to extract unsafe tar member: {member.name}") + tf.extractall(destination_dir) + + +def _download_hf_file( + repo_id: str, + filename: str, + download_dir: str | Path, + repo_type: str = "dataset", +) -> Path: + """Download one file from the Hugging Face Hub into download_dir and return its local path.""" + from huggingface_hub import hf_hub_download + + download_dir = Path(download_dir) + download_dir.mkdir(parents=True, exist_ok=True) + return Path( + hf_hub_download( + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + local_dir=download_dir, + ) + ) + + +class CoSpyDataset(Dataset): + """CO-SPY-Bench test-only dataset (Cheng et al., CVPR 2025). + + The benchmark lives in two HuggingFace dataset repos: + + * Synthetic images: ``ruojiruoli/Co-Spy-Bench`` as + ``{source_dataset}/{model}/images_with_metadata.tar.gz`` archives, + with five real-image source datasets and 22 generator models. + * Real images: ``ruojiruoli/Co-Spy-Misc`` as a single + ``real_image_examples.zip`` archive (one subfolder per source). + + After download, ``prepare_split`` restructures the data into the exact + layout used by ``CNNDetectionDataset`` (``cnn_synth_testset``):: + + /test/ + ├── __/ + │ ├── 0_real/*.png (hardlinks into a single shared real pool) + │ └── 1_fake/*.png (moved from the extracted tarball) + ├── ... + └── (5 * 22 = 110 class folders) + + The class name uses ``__`` (separator ``__``), which gives + up to 110 unique ``folder_name``s matching the granularity of Table 2 in + the paper. Real images are physically stored once per source under + ``/test/_real_pool//`` and exposed inside each per-model + ``0_real/`` directory via ``os.link`` (zero extra disk usage); a + ``shutil.copy2`` fallback is used on filesystems that do not support + hardlinks. With this layout, label and folder extraction reuse the + same convention as ``CNNDetectionDataset``: the leaf folder name + (``0_real``/``1_fake``) drives the label and the grandparent folder + (``__``) drives ``folder_name``. + + Note: ``ruojiruoli/Co-Spy-Misc/real_image_examples.zip`` is a *partial* + release that ships reals for ``CC3M``/``SBU``/``TextCaps`` only (with + that exact casing). Reals for ``mscoco`` and ``flickr`` must be obtained + from their original sources and placed under + ``/test/_real_pool//``; class folders without reals will + contain only ``1_fake/``. + """ + + repo_id = "ruojiruoli/Co-Spy-Bench" + real_repo_id = "ruojiruoli/Co-Spy-Misc" + real_filename = "real_image_examples.zip" + + default_source_datasets: tuple[str, ...] = ( + "mscoco", + "flickr", + "cc3m", + "textcaps", + "sbu", + ) + default_generator_models: tuple[str, ...] = ( + "CompVis@ldm-text2im-large-256", + "CompVis@stable-diffusion-v1-4", + "runwayml@stable-diffusion-v1-5", + "segmind@SSD-1B", + "segmind@tiny-sd", + "segmind@SegMoE-SD-4x2-v0", + "segmind@small-sd", + "stabilityai@stable-diffusion-2-1", + "stabilityai@stable-diffusion-3-medium-diffusers", + "stabilityai@sdxl-turbo", + "stabilityai@stable-diffusion-2", + "stabilityai@stable-diffusion-xl-base-1.0", + "playgroundai@playground-v2.5-1024px-aesthetic", + "playgroundai@playground-v2-1024px-aesthetic", + "playgroundai@playground-v2-512px-base", + "playgroundai@playground-v2-256px-base", + "PixArt-alpha@PixArt-XL-2-1024-MS", + "PixArt-alpha@PixArt-XL-2-512x512", + "latent-consistency@lcm-lora-sdxl", + "latent-consistency@lcm-lora-sdv1-5", + "black-forest-labs@FLUX.1-schnell", + "black-forest-labs@FLUX.1-dev", + ) + + CLASS_SEPARATOR = "__" + REAL_LABEL_DIR = "0_real" + FAKE_LABEL_DIR = "1_fake" + REAL_POOL_DIR = "_real_pool" + + def __init__( + self, + root_dir: str | Path, + split: Literal["test"] = "test", + transform: AIGITransformations | None = None, + prefetch: bool = False, + source_datasets: tuple[str, ...] | list[str] | None = None, + generator_models: tuple[str, ...] | list[str] | None = None, + include_real: bool = True, + repo_id: str | None = None, + real_repo_id: str | None = None, + real_filename: str | None = None, + ): + super().__init__() + if split != "test": + raise ValueError("CoSpyDataset only supports the 'test' split.") + + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.repo_id = repo_id or self.repo_id + self.real_repo_id = real_repo_id or self.real_repo_id + self.real_filename = real_filename or self.real_filename + self.source_datasets = tuple(source_datasets) if source_datasets else self.default_source_datasets + self.generator_models = tuple(generator_models) if generator_models else self.default_generator_models + self.include_real = include_real + self.destination_dir = self.root_dir / self.split + + if prefetch and not _has_images(self.destination_dir): + print("Downloading CO-SPY-Bench test split from Hugging Face...") + self._download_data() + + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["test"] = "test", + source_datasets: tuple[str, ...] | list[str] | None = None, + generator_models: tuple[str, ...] | list[str] | None = None, + include_real: bool = True, + repo_id: str = "ruojiruoli/Co-Spy-Bench", + real_repo_id: str = "ruojiruoli/Co-Spy-Misc", + real_filename: str = "real_image_examples.zip", + force: bool = False, + keep_downloads: bool = False, + ) -> None: + """Download CO-SPY-Bench and lay it out in CNNDetection-style ``test/`` tree.""" + if split != "test": + raise ValueError("CoSpyDataset only supports the 'test' split.") + + from huggingface_hub import hf_hub_download, snapshot_download + + root_dir = Path(root_dir) + destination_dir = root_dir / split + if not force and _has_images(destination_dir): + return + if force: + shutil.rmtree(destination_dir, ignore_errors=True) + destination_dir.mkdir(parents=True, exist_ok=True) + + sources = tuple(source_datasets) if source_datasets else cls.default_source_datasets + models = tuple(generator_models) if generator_models else cls.default_generator_models + + bench_download_dir = root_dir / "_downloads" / "Co-Spy-Bench" + patterns = [ + f"{ds}/{model}/images_with_metadata.tar.gz" + for ds in sources + for model in models + ] + snapshot_download( + repo_id=repo_id, + repo_type="dataset", + local_dir=bench_download_dir, + allow_patterns=patterns, + ) + + for tar_path in sorted(bench_download_dir.rglob("images_with_metadata.tar.gz")): + rel = tar_path.relative_to(bench_download_dir) + if len(rel.parts) < 2: + continue + source_name = rel.parts[0] + extract_target = bench_download_dir / source_name + _safe_extract_tar(tar_path, extract_target) + + real_extracted_root: Path | None = None + if include_real: + real_download_dir = root_dir / "_downloads" / "Co-Spy-Misc" + real_download_dir.mkdir(parents=True, exist_ok=True) + real_path = hf_hub_download( + repo_id=real_repo_id, + filename=real_filename, + repo_type="dataset", + local_dir=real_download_dir, + ) + real_extract_dir = real_download_dir / "extracted" + _safe_extract_zip(real_path, real_extract_dir) + real_extracted_root = cls._find_real_root(real_extract_dir, sources) + + cls._restructure_to_cnndet_layout( + synthetic_root=bench_download_dir, + real_root=real_extracted_root, + destination_dir=destination_dir, + sources=sources, + models=models, + ) + + if not _has_images(destination_dir): + raise RuntimeError( + "CO-SPY-Bench download did not produce any recognizable images. " + f"Check that the HuggingFace repo {repo_id!r} is reachable and " + "that at least one (source_dataset, model) pair has been requested." + ) + + if not keep_downloads: + shutil.rmtree(root_dir / "_downloads", ignore_errors=True) + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + source_datasets: tuple[str, ...] | list[str] | None = None, + generator_models: tuple[str, ...] | list[str] | None = None, + include_real: bool = True, + repo_id: str = "ruojiruoli/Co-Spy-Bench", + real_repo_id: str = "ruojiruoli/Co-Spy-Misc", + real_filename: str = "real_image_examples.zip", + force: bool = False, + keep_downloads: bool = False, + ) -> None: + """Download CO-SPY-Bench as a test-only dataset under ``root_dir/test``.""" + cls.prepare_split( + root_dir=root_dir, + split="test", + source_datasets=source_datasets, + generator_models=generator_models, + include_real=include_real, + repo_id=repo_id, + real_repo_id=real_repo_id, + real_filename=real_filename, + force=force, + keep_downloads=keep_downloads, + ) + + @staticmethod + def _case_insensitive_subdir(parent: Path, name: str) -> Path | None: + """Return ``parent/`` whose lowercase name equals ``name.lower()``.""" + if not parent.is_dir(): + return None + target = name.lower() + for entry in parent.iterdir(): + if entry.is_dir() and entry.name.lower() == target: + return entry + return None + + @classmethod + def _find_real_root(cls, real_extract_dir: Path, sources: tuple[str, ...]) -> Path | None: + """Locate the directory that holds per-source real-image subfolders. + + The upstream ``real_image_examples.zip`` only ships reals for a subset + of the five Co-Spy sources (currently ``CC3M``/``SBU``/``TextCaps``) + and uses an inconsistent casing for the per-source folder names, so + the search is case-insensitive and accepts a partial source set. + """ + candidates: list[Path] = [ + real_extract_dir / "real_image_examples", + real_extract_dir, + ] + candidates.extend(p for p in real_extract_dir.rglob("real_image_examples") if p.is_dir()) + for candidate in candidates: + if not candidate.is_dir(): + continue + if any(cls._case_insensitive_subdir(candidate, src) is not None for src in sources): + return candidate + for candidate in real_extract_dir.rglob("*"): + if not candidate.is_dir(): + continue + if any(cls._case_insensitive_subdir(candidate, src) is not None for src in sources): + return candidate + return None + + @classmethod + def _restructure_to_cnndet_layout( + cls, + synthetic_root: Path, + real_root: Path | None, + destination_dir: Path, + sources: tuple[str, ...], + models: tuple[str, ...], + ) -> None: + """Move synthetic + hardlink real images into ``__/{0_real,1_fake}``.""" + destination_dir.mkdir(parents=True, exist_ok=True) + + real_pool_dir = destination_dir / cls.REAL_POOL_DIR + if real_root is not None: + for source in sources: + source_real_dir = cls._case_insensitive_subdir(real_root, source) + if source_real_dir is None: + continue + pool_target = real_pool_dir / source + pool_target.mkdir(parents=True, exist_ok=True) + for img in _iter_images(source_real_dir): + target = cls._dedup_name(pool_target / img.name, img) + if target.exists(): + continue + shutil.move(str(img), str(target)) + + report: list[tuple[str, str, int, int]] = [] + for source in sources: + for model in models: + model_synthetic_dir = synthetic_root / source / model + model_images = ( + list(_iter_images(model_synthetic_dir)) + if model_synthetic_dir.is_dir() + else [] + ) + source_real_pool = real_pool_dir / source + pool_images = ( + list(_iter_images(source_real_pool)) + if source_real_pool.is_dir() + else [] + ) + if not model_images and not pool_images: + continue + + class_dir = destination_dir / f"{source}{cls.CLASS_SEPARATOR}{model}" + if model_images: + fake_dir = class_dir / cls.FAKE_LABEL_DIR + fake_dir.mkdir(parents=True, exist_ok=True) + for img in model_images: + target = cls._dedup_name(fake_dir / img.name, img) + if target.exists(): + continue + shutil.move(str(img), str(target)) + if pool_images: + real_dir = class_dir / cls.REAL_LABEL_DIR + real_dir.mkdir(parents=True, exist_ok=True) + for img in pool_images: + target = cls._dedup_name(real_dir / img.name, img) + if target.exists(): + continue + cls._hardlink_or_copy(img, target) + + report.append((source, model, len(model_images), len(pool_images))) + + if report: + sources_missing_reals = sorted({src for src, _, _, n_real in report if n_real == 0}) + sources_with_reals = sorted({src for src, _, _, n_real in report if n_real > 0}) + print(f"[CoSpyDataset] reals available for sources : {sources_with_reals}") + if sources_missing_reals: + print( + f"[CoSpyDataset] reals MISSING for sources : {sources_missing_reals} " + f"(class folders will have only '{cls.FAKE_LABEL_DIR}/'; provide reals " + f"under '{real_pool_dir}//' to include them)" + ) + + @staticmethod + def _hardlink_or_copy(src: Path, dst: Path) -> None: + """Create ``dst`` as a hardlink to ``src``; fall back to a copy on failure.""" + try: + os.link(src, dst) + except OSError: + shutil.copy2(src, dst) + + @staticmethod + def _dedup_name(target: Path, source: Path) -> Path: + """Append a short SHA-1 prefix to ``target`` when its name already exists.""" + if not target.exists(): + return target + digest = hashlib.sha1(str(source).encode("utf-8")).hexdigest()[:10] + return target.with_name(f"{target.stem}_{digest}{target.suffix}") + + def _download_data(self, force: bool = False) -> None: + self.prepare_split( + root_dir=self.root_dir, + split=self.split, + source_datasets=self.source_datasets, + generator_models=self.generator_models, + include_real=self.include_real, + repo_id=self.repo_id, + real_repo_id=self.real_repo_id, + real_filename=self.real_filename, + force=force, + ) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name: + return 0 + if "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + def _load_data(self): + images, labels, folder_names = [], [], [] + split_dir = self.destination_dir + pool_marker = self.REAL_POOL_DIR + for filepath in _iter_images(split_dir): + if pool_marker in filepath.parts: + continue + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append( + filepath.parent.parent.name + if filepath.parent.parent != split_dir.parent + else self.split + ) + return images, labels, folder_names + + +class GenImageDataset(Dataset): + google_drive_folder_url = "https://drive.google.com/drive/folders/1jGt10bwTbhEZuGXLyvrCuxOI0cBqQ1FS" + hf_repo_id = None + hf_subdir = None + split_aliases = { + "train": "train", + "val": "test", + "test": "test", + } + source_split_markers = { + "train": ("train",), + "test": ("val", "validation", "test"), + } + class_dir_to_label = { + "nature": "real", + "real": "real", + "ai": "fake", + "fake": "fake", + "false": "fake", + } + + def __init__(self, root_dir: str | Path, split: Literal["train", "val", "test"], transform: AIGITransformations | None = None, prefetch: bool = False): + super().__init__() + self.root_dir = Path(root_dir) + self.requested_split = split + self.split = self.split_aliases[split] + self.transform = transform + self.prefetch = prefetch + self.destination_dir = self.root_dir / self.split + if prefetch and not _has_images(self.destination_dir): + print(f"Downloading {self.split} split from Google Drive...") + self._download_data() + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + @staticmethod + def _tokens(path: Path) -> list[str]: + tokens: list[str] = [] + for part in path.parts: + tokens.extend(token for token in re.split(r"[^a-z0-9]+", part.lower()) if token) + return tokens + + @classmethod + def download_files( + cls, + download_dir: str | Path, + url: str | None = None, + force: bool = False, + ) -> Path: + """Download the shared GenImage Google Drive folder into download_dir.""" + download_dir = Path(download_dir) + if download_dir.exists() and any(download_dir.iterdir()) and not force: + return download_dir + + if force: + shutil.rmtree(download_dir, ignore_errors=True) + download_dir.mkdir(parents=True, exist_ok=True) + + if cls.hf_repo_id: + from huggingface_hub import hf_hub_download, list_repo_files + + repo_files = [f for f in list_repo_files(cls.hf_repo_id, repo_type="dataset") if not f.endswith("/")] + if cls.hf_subdir: + prefix = str(cls.hf_subdir).rstrip("/") + "/" + repo_files = [f for f in repo_files if f.startswith(prefix)] + if not repo_files: + raise RuntimeError( + f"No files found in HF dataset {cls.hf_repo_id!r} under subdir {cls.hf_subdir!r}." + ) + for filename in sorted(repo_files): + hf_hub_download( + repo_id=cls.hf_repo_id, + filename=filename, + repo_type="dataset", + local_dir=str(download_dir), + ) + return download_dir + + try: + import gdown + except ImportError as exc: + raise ImportError( + "Downloading the GenImage Google Drive folder requires `gdown`. " + "Install it with `pip install gdown`." + ) from exc + + gdown.download_folder( + url=url or cls.google_drive_folder_url, + output=str(download_dir), + quiet=False, + use_cookies=False, + ) + return download_dir + + @classmethod + def _extract_downloaded_archives(cls, download_dir: Path) -> None: + for zip_path in sorted(download_dir.rglob("*.zip")): + extract_dir = zip_path.with_suffix("") + if _has_images(extract_dir): + continue + if zip_path.with_suffix(".z01").exists(): + cls._extract_spanned_zip(zip_path, extract_dir) + else: + _safe_extract_zip(zip_path, extract_dir) + + @staticmethod + def _extract_spanned_zip(zip_path: Path, destination_dir: Path) -> None: + """Extract a multi-part (spanned) zip (.z01..+.zip) using 7z.""" + seven_zip = shutil.which("7z") or shutil.which("7za") + if seven_zip is None: + raise RuntimeError( + "Extracting multi-part (spanned) zip archives requires the `7z` command. Install p7zip-full." + ) + destination_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [seven_zip, "x", str(zip_path), f"-o{destination_dir}", "-y"], + cwd=zip_path.parent, + check=True, + ) + + @classmethod + def _source_split_for_path(cls, filepath: Path, destination_split: Literal["train", "test"]) -> bool: + tokens = cls._tokens(filepath.parent) + return any(marker in tokens for marker in cls.source_split_markers[destination_split]) + + @classmethod + def _label_dir_for_path(cls, filepath: Path) -> str | None: + tokens = cls._tokens(filepath.parent) + for source_name, label_dir in cls.class_dir_to_label.items(): + if source_name in tokens: + return label_dir + return None + + @classmethod + def _relative_image_path(cls, filepath: Path) -> Path: + parts = list(filepath.parts) + lower_parts = [part.lower() for part in parts] + marker_index = -1 + for idx, part in enumerate(lower_parts[:-1]): + tokens = cls._tokens(Path(part)) + if any(token in cls.source_split_markers["train"] + cls.source_split_markers["test"] for token in tokens): + marker_index = idx + if any(token in cls.class_dir_to_label for token in tokens): + marker_index = idx + relative_parts = parts[marker_index + 1 :] if marker_index >= 0 else [filepath.name] + return Path(*relative_parts) + + @staticmethod + def _deduplicated_destination(destination_path: Path, source_path: Path) -> Path: + if not destination_path.exists(): + return destination_path + + digest = hashlib.sha1(str(source_path).encode("utf-8")).hexdigest()[:10] + return destination_path.with_name(f"{destination_path.stem}_{digest}{destination_path.suffix}") + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + url: str | None = None, + force: bool = False, + keep_downloads: bool = True, + ) -> None: + """Download GenImage and normalize it to root/train and root/test.""" + root_dir = Path(root_dir) + train_dir = root_dir / "train" + test_dir = root_dir / "test" + if not force and _has_images(train_dir) and _has_images(test_dir): + return + + if force: + shutil.rmtree(train_dir, ignore_errors=True) + shutil.rmtree(test_dir, ignore_errors=True) + + download_dir = root_dir / "_downloads" / "GenImage" + cls.download_files(download_dir, url=url, force=force) + cls._extract_downloaded_archives(download_dir) + + copied = {"train": 0, "test": 0} + for destination_split in ("train", "test"): + destination_split_dir = root_dir / destination_split + destination_split_dir.mkdir(parents=True, exist_ok=True) + + for source_path in _iter_images(download_dir): + source_relative_path = source_path.relative_to(download_dir) + if not cls._source_split_for_path(source_relative_path, destination_split): + continue + + label_dir = cls._label_dir_for_path(source_relative_path) + if label_dir is None: + continue + + relative_path = cls._relative_image_path(source_relative_path) + destination_path = cls._deduplicated_destination( + destination_split_dir / label_dir / relative_path, + source_path, + ) + destination_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, destination_path) + copied[destination_split] += 1 + + missing = [split for split, count in copied.items() if count == 0] + if missing: + raise RuntimeError( + "GenImage download did not contain recognizable images for " + f"{', '.join(missing)}. Expected train/val split folders and ai/nature class folders." + ) + + if not keep_downloads: + shutil.rmtree(root_dir / "_downloads", ignore_errors=True) + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["train", "val", "test"], + url: str | None = None, + force: bool = False, + keep_downloads: bool = True, + ) -> None: + """Download and normalize GenImage; val is stored as the test split.""" + destination_split = cls.split_aliases[split] + destination_dir = Path(root_dir) / destination_split + if not force and _has_images(destination_dir): + return + cls.prepare_dataset(root_dir, url=url, force=force, keep_downloads=keep_downloads) + + def _download_data(self, force: bool = False) -> None: + """Download and normalize this dataset split from Google Drive.""" + self.prepare_split( + root_dir=self.root_dir, + split=self.requested_split, + url=self.google_drive_folder_url, + force=force, + ) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + def _iter_images(self, directory: Path): + return _iter_images(directory) + + def _load_data(self): + images, labels, folder_names = [], [], [] + split_dir = self.destination_dir + for filepath in self._iter_images(split_dir): + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append(self.split) + return images, labels, folder_names + + @staticmethod + def _get_label(filepath: Path) -> int: + names = [part.lower() for part in filepath.parent.parts] + if "real" in names: + return 0 + if "fake" in names or "false" in names: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + +class CNNDetectionDataset(Dataset): + google_drive_folder_url = "https://drive.google.com/drive/u/0/folders/1RwCSaraEUctIwFgoQXWMKFvW07gM80_3" + archive_name_markers = { + "train": ("progan_train", "train"), + "val": ("progan_val", "val"), + "test": ("cnn_synth_testset", "test"), + } + + def __init__(self, root_dir: str | Path, split: Literal["train", "val", "test"], transform: AIGITransformations | None = None, prefetch: bool = False): + super().__init__() + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.prefetch = prefetch + self.destination_dir = self.root_dir / self.split + if prefetch and not _has_images(self.destination_dir): + print(f"Downloading {self.split} split from Google Drive...") + self._download_data() + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + @classmethod + def download_archives( + cls, + archive_dir: str | Path, + url: str | None = None, + force: bool = False, + ) -> Path: + """Download the shared CNNDetection Google Drive folder into archive_dir.""" + archive_dir = Path(archive_dir) + if archive_dir.exists() and any(archive_dir.iterdir()) and not force: + return archive_dir + + archive_dir.mkdir(parents=True, exist_ok=True) + try: + import gdown + except ImportError as exc: + raise ImportError( + "Downloading a Google Drive folder requires `gdown`. " + "Install it with `pip install gdown`." + ) from exc + + gdown.download_folder( + url=url or cls.google_drive_folder_url, + output=str(archive_dir), + quiet=False, + use_cookies=False, + ) + return archive_dir + + @classmethod + def _find_split_archives( + cls, + archive_dir: Path, + split: Literal["train", "val", "test"], + ) -> list[Path]: + markers = cls.archive_name_markers[split] + archive_suffixes = (".zip", ".7z", ".001") + archives = [ + p + for p in archive_dir.rglob("*") + if p.is_file() + and p.name.lower().endswith(archive_suffixes) + and any(marker in p.name.lower() for marker in markers) + ] + + if split == "train": + zip_archives = [p for p in archives if p.suffix.lower() == ".zip"] + if zip_archives: + return sorted(zip_archives) + first_7z_parts = [p for p in archives if p.name.lower().endswith(".7z.001")] + if first_7z_parts: + return sorted(first_7z_parts) + + return sorted(archives) + + @staticmethod + def _extract_7z(archive_path: Path, destination_dir: Path) -> None: + seven_zip = shutil.which("7z") or shutil.which("7za") + if seven_zip is None: + raise RuntimeError("Extracting .7z archives requires the `7z` command. Install p7zip-full.") + + destination_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [seven_zip, "x", str(archive_path), f"-o{destination_dir}", "-y"], + cwd=archive_path.parent, + check=True, + ) + + @classmethod + def _extract_archive(cls, archive_path: Path, destination_dir: Path) -> None: + if archive_path.suffix.lower() == ".zip": + _safe_extract_zip(archive_path, destination_dir) + return + + if archive_path.name.lower().endswith(".7z") or archive_path.name.lower().endswith(".7z.001"): + cls._extract_7z(archive_path, destination_dir) + for zip_path in destination_dir.rglob("*.zip"): + _safe_extract_zip(zip_path, destination_dir) + return + + raise RuntimeError(f"Unsupported archive type: {archive_path}") + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["train", "val", "test"], + force: bool = False, + keep_archives: bool = True, + ) -> None: + """Download and extract one CNNDetection split under root_dir/split.""" + root_dir = Path(root_dir) + destination_dir = root_dir / split + if not force and _has_images(destination_dir): + return + + archive_dir = root_dir / "_archives" / "CNNDetection" + cls.download_archives(archive_dir) + archive_paths = cls._find_split_archives(archive_dir, split) + if not archive_paths: + raise RuntimeError(f"No archive found for CNNDetection {split!r} split in {archive_dir}") + + destination_dir.mkdir(parents=True, exist_ok=True) + for archive_path in archive_paths: + cls._extract_archive(archive_path, destination_dir) + + if not keep_archives: + shutil.rmtree(archive_dir, ignore_errors=True) + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + splits: tuple[Literal["train", "val", "test"], ...] = ("train", "val", "test"), + force: bool = False, + keep_archives: bool = True, + ) -> None: + """Download and extract the requested CNNDetection splits.""" + for split in splits: + cls.prepare_split(root_dir, split, force=force, keep_archives=keep_archives) + + def _download_data(self, force: bool = False) -> None: + """Download and extract this dataset split under root_dir/split.""" + self.prepare_split(self.root_dir, self.split, force=force) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name: + return 0 + if "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + def _iter_images(self, directory: Path): + return _iter_images(directory) + + def _load_data(self): + images, labels, folder_names = [], [], [] + split_dir = self.destination_dir + for filepath in self._iter_images(split_dir): + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append(filepath.parent.parent.name if filepath.parent.parent != split_dir.parent else self.split) + return images, labels, folder_names + + +class CNNDetectionDatasetModule: + def __init__(self, config): + self.config = config + train_list = list(config.DatasetConfig.train_transform_list) + eval_list = list(config.DatasetConfig.eval_transform_list) + self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + self.train_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "train", self.train_transform, prefetch=prefetch) + self.val_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "val", self.eval_transform, prefetch=prefetch) + self.test_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch) + + def train_dataloader(self): + return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def val_dataloader(self): + return DataLoader(self.val_dataset, batch_size=self.config.DatasetConfig.val_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class GenImageDatasetModule: + def __init__(self, config): + self.config = config + train_list = list(config.DatasetConfig.train_transform_list) + eval_list = list(config.DatasetConfig.eval_transform_list) + self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + source_url = config.DatasetConfig.get("source_url", GenImageDataset.google_drive_folder_url) + GenImageDataset.google_drive_folder_url = source_url + GenImageDataset.hf_repo_id = config.DatasetConfig.get("hf_repo_id", None) + GenImageDataset.hf_subdir = config.DatasetConfig.get("hf_subdir", None) + self.train_dataset = GenImageDataset(config.DatasetConfig.root_dir, "train", self.train_transform, prefetch=prefetch) + self.val_dataset = GenImageDataset(config.DatasetConfig.root_dir, "val", self.eval_transform, prefetch=prefetch) + self.test_dataset = GenImageDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch) + + def train_dataloader(self): + return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def val_dataloader(self): + return DataLoader(self.val_dataset, batch_size=self.config.DatasetConfig.val_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class AIGIBenchDataset(Dataset): + def __init__( + self, + root_dir: str | Path, + split: Literal["train", "val", "test"], + setting: Literal["Setting-I", "Setting-II"], + transform: AIGITransformations | None = None, + prefetch: bool = False, + ): + super().__init__() + self.root_dir = Path(root_dir) + self.split = split + self.setting = setting + self.transform = transform + self.unique_folder_names = ["train"] + self.destination_dir = self.root_dir / self.split + self.huggingface_path = "https://huggingface.co/datasets/HorizonTEL/AIGIBench/tree/main" + self.huggingface_repo_id = "HorizonTEL/AIGIBench" + if prefetch: + if _has_images(self.destination_dir): + print(f"on prefetch_hf mode, {self.destination_dir} already exists") + else: + print(f"Downloading {self.split} split from Hugging Face...") + self._download_data() + if self.split != "train": + self.unique_folder_names += [d.name for d in self.destination_dir.iterdir() if d.is_dir()] if self.destination_dir.exists() else [] + self.images, self.labels, self.folder_names = self._load_data() + + @staticmethod + def _split_has_images(directory: Path) -> bool: + return _has_images(directory) + + @classmethod + def prepare_split( + cls, + destination_dir: str | Path, + split: Literal["train", "val", "test"], + repo_id: str = "HorizonTEL/AIGIBench", + force: bool = False, + ) -> None: + """Download all zip archives for one split from the Hub and extract under root_dir.""" + from huggingface_hub import hf_hub_download, list_repo_files + + destination_dir = Path(destination_dir) + destination_dir.mkdir(parents=True, exist_ok=True) + prefix = f"{split}/" + files = list_repo_files(repo_id, repo_type="dataset") + zips = sorted(f for f in files if f.startswith(prefix) and f.lower().endswith(".zip")) + if not zips: + raise RuntimeError(f"No .zip files under {prefix!r} in dataset {repo_id!r}") + + for rel in zips: + stem = Path(rel).stem + target_dir = destination_dir / stem + if not force and target_dir.is_dir() and cls._split_has_images(target_dir): + continue + + local_zip = hf_hub_download( + repo_id=repo_id, + filename=rel, + repo_type="dataset", + ) + _safe_extract_zip(local_zip, destination_dir) + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + splits: tuple[Literal["train", "val", "test"], ...] = ("train", "val", "test"), + repo_id: str = "HorizonTEL/AIGIBench", + force: bool = False, + ) -> None: + """Download and extract AIGIBench split archives under root_dir/train, root_dir/val, root_dir/test.""" + root_dir = Path(root_dir) + for split in splits: + cls.prepare_split( + destination_dir=root_dir / split, + split=split, + repo_id=repo_id, + force=force, + ) + + def _download_data(self, force: bool = False) -> None: + """Download and extract this dataset split from Hugging Face (skips subsets that already contain images).""" + self.prepare_split( + destination_dir=self.destination_dir, + split=self.split, + repo_id=self.huggingface_repo_id, + force=force, + ) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path, folder_names: list[str] | None = None) -> int: + name = filepath.parent.name + if "real" in name: + return 0 + elif "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + def _iter_images(self, directory: Path): + return _iter_images(directory) + + def _load_data(self): + images, labels, folder_names = [], [], [] + split_dir = self.destination_dir + + for filepath in self._iter_images(split_dir): + if self.split == "train" and self.setting == "Setting-I": + if "sdv1.4" in str(filepath): + continue + + images.append(filepath) + labels.append(self._get_label(filepath, self.unique_folder_names)) + folder_names.append( + filepath.parent.parent.name if self.split != "train" else "train" + ) + + return images, labels, folder_names + + +class DiTFakeDataset(Dataset): + repo_id = "lioooox/DiTFake" + + def __init__( + self, + root_dir: str | Path, + split: Literal["test"], + transform: AIGITransformations | None = None, + prefetch: bool = False, + repo_id: str | None = None, + ): + super().__init__() + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.repo_id = repo_id or self.repo_id + self.destination_dir = self.root_dir / self.split + self._normalize_existing_layout() + + if prefetch and not _has_images(self.destination_dir): + print("Downloading DiTFake test split from Hugging Face...") + self._download_data() + self.destination_dir = self.root_dir / self.split + self._normalize_existing_layout() + + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + def _normalize_existing_layout(self) -> None: + nested_split_dir = self.root_dir / "DiTFake" / self.split + if _has_images(nested_split_dir) and not _has_images(self.destination_dir): + if self.destination_dir.exists(): + shutil.rmtree(self.destination_dir) + self.destination_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(nested_split_dir), str(self.destination_dir)) + nested_root = self.root_dir / "DiTFake" + if nested_root.exists() and not any(nested_root.rglob("*")): + shutil.rmtree(nested_root, ignore_errors=True) + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["test"] = "test", + repo_id: str = "lioooox/DiTFake", + force: bool = False, + ) -> None: + """Download one DiTFake split subtree into root_dir/split.""" + from huggingface_hub import snapshot_download + + root_dir = Path(root_dir) + destination_dir = root_dir / split + if not force and _has_images(destination_dir): + return + + root_dir.mkdir(parents=True, exist_ok=True) + download_dir = root_dir / "_downloads" / "DiTFake" + snapshot_download( + repo_id=repo_id, + repo_type="dataset", + local_dir=download_dir, + allow_patterns=[f"DiTFake/{split}/**"], + ) + downloaded_split_dir = download_dir / "DiTFake" / split + if downloaded_split_dir.exists() and downloaded_split_dir != destination_dir: + if destination_dir.exists() and (force or not _has_images(destination_dir)): + shutil.rmtree(destination_dir) + destination_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(downloaded_split_dir), str(destination_dir)) + shutil.rmtree(root_dir / "_downloads", ignore_errors=True) + nested_root = root_dir / "DiTFake" + if nested_root.exists(): + shutil.rmtree(nested_root, ignore_errors=True) + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + repo_id: str = "lioooox/DiTFake", + force: bool = False, + ) -> None: + """Download DiTFake test split under root_dir.""" + cls.prepare_split(root_dir=root_dir, split="test", repo_id=repo_id, force=force) + + def _download_data(self, force: bool = False) -> None: + self.prepare_split( + root_dir=self.root_dir, + split=self.split, + repo_id=self.repo_id, + force=force, + ) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name: + return 0 + if "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + def _load_data(self): + images, labels, folder_names = [], [], [] + for filepath in _iter_images(self.destination_dir): + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append(filepath.parent.parent.name) + return images, labels, folder_names + + + +class FreqNetDetectionDataset(Dataset): + google_drive_folder_url = "https://drive.google.com/drive/folders/11E0Knf9J1qlv2UuTnJSOFUjIIi90czSj" + + def __init__( + self, + root_dir: str | Path, + split: Literal["test"], + transform: AIGITransformations | None = None, + prefetch: bool = False, + source_url: str | None = None, + ): + super().__init__() + if split != "test": + raise ValueError("FreqNetDetectionDataset only supports the test split.") + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.prefetch = prefetch + self.source_url = source_url or self.google_drive_folder_url + self.destination_dir = self.root_dir / self.split + if prefetch and not _has_images(self.destination_dir): + print("Downloading FreqNetDetection test split from Google Drive...") + self._download_data() + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + @classmethod + def download_archives( + cls, + archive_dir: str | Path, + url: str | None = None, + force: bool = False, + ) -> Path: + """Download the shared FreqNetDetection Google Drive folder into archive_dir.""" + archive_dir = Path(archive_dir) + if archive_dir.exists() and any(archive_dir.iterdir()) and not force: + return archive_dir + + if force: + shutil.rmtree(archive_dir, ignore_errors=True) + archive_dir.mkdir(parents=True, exist_ok=True) + + try: + import gdown + except ImportError as exc: + raise ImportError( + "Downloading the FreqNetDetection Google Drive folder requires `gdown`. " + "Install it with `pip install gdown`." + ) from exc + + gdown.download_folder( + url=url or cls.google_drive_folder_url, + output=str(archive_dir), + quiet=False, + use_cookies=False, + ) + return archive_dir + + @staticmethod + def _archive_paths(archive_dir: Path) -> list[Path]: + archive_suffixes = (".tar.gz", ".tgz", ".tar") + return sorted( + p + for p in archive_dir.rglob("*") + if p.is_file() and p.name.lower().endswith(archive_suffixes) + ) + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["test"] = "test", + source_url: str | None = None, + force: bool = False, + keep_archives: bool = True, + ) -> None: + """Download FreqNetDetection generator archives and extract them under root_dir/test.""" + if split != "test": + raise ValueError("FreqNetDetectionDataset only supports the test split.") + + root_dir = Path(root_dir) + destination_dir = root_dir / split + if _has_images(destination_dir) and not force: + return + if force: + shutil.rmtree(destination_dir, ignore_errors=True) + + archive_dir = root_dir / "_archives" / "FreqNetDetection" + cls.download_archives(archive_dir, url=source_url, force=force) + archive_paths = cls._archive_paths(archive_dir) + if not archive_paths: + raise RuntimeError(f"No .tar archives found for FreqNetDetection in {archive_dir}") + + destination_dir.mkdir(parents=True, exist_ok=True) + for archive_path in archive_paths: + _safe_extract_tar(archive_path, destination_dir) + + if not _has_images(destination_dir): + raise RuntimeError("FreqNetDetection archives did not contain recognizable images.") + if not keep_archives: + shutil.rmtree(archive_dir, ignore_errors=True) + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + source_url: str | None = None, + force: bool = False, + keep_archives: bool = True, + ) -> None: + """Download FreqNetDetection as a test-only dataset under root_dir/test.""" + cls.prepare_split( + root_dir=root_dir, + split="test", + source_url=source_url, + force=force, + keep_archives=keep_archives, + ) + + def _download_data(self, force: bool = False) -> None: + self.prepare_split( + root_dir=self.root_dir, + split=self.split, + source_url=self.source_url, + force=force, + ) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name: + return 0 + if "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + def _load_data(self): + images, labels, folder_names = [], [], [] + for filepath in _iter_images(self.destination_dir): + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append(filepath.parent.parent.name) + return images, labels, folder_names + + +class UniversialFakeDetectionDataset(Dataset): + source_url = "https://drive.google.com/file/d/1FXlGIRh_Ud3cScMgSVDbEWmPDmjcrm1t/view" + + def __init__( + self, + root_dir: str | Path, + split: Literal["test"], + transform: AIGITransformations | None = None, + prefetch: bool = False, + source_url: str | None = None, + ): + super().__init__() + if split != "test": + raise ValueError("UniversialFakeDetectionDataset only supports the test split.") + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.prefetch = prefetch + self.source_url = source_url or self.source_url + self.destination_dir = self.root_dir / self.split + if prefetch and not _has_images(self.destination_dir): + print(f"Downloading {self.split} split from Google Drive...") + self._download_data() + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _flatten_extracted_tree(extract_dir: Path, destination_dir: Path) -> None: + source_dir = extract_dir / "diffusion_models" + if not source_dir.exists(): + nested = [p for p in extract_dir.rglob("diffusion_models") if p.is_dir()] + source_dir = nested[0] if nested else extract_dir + container_names = {"diffusion_datasets", "diffusion_dataset", "datasets", "data"} + child_dirs = [p for p in source_dir.iterdir() if p.is_dir()] + if len(child_dirs) == 1 and child_dirs[0].name.lower() in container_names: + source_dir = child_dirs[0] + + destination_dir.mkdir(parents=True, exist_ok=True) + for child in source_dir.iterdir(): + target = destination_dir / child.name + if target.exists(): + if target.is_dir(): + shutil.rmtree(target) + else: + target.unlink() + shutil.move(str(child), str(target)) + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["test"] = "test", + source_url: str | None = None, + force: bool = False, + keep_downloads: bool = False, + ) -> None: + """Download the Universal Fake Detection zip and flatten diffusion_models under root_dir/test.""" + if split != "test": + raise ValueError("UniversialFakeDetectionDataset only supports the test split.") + + root_dir = Path(root_dir) + destination_dir = root_dir / split + if _has_images(destination_dir) and not force: + return + if force: + shutil.rmtree(destination_dir, ignore_errors=True) + + import gdown + + download_dir = root_dir / "_downloads" / "UniversialFakeDetection" + extract_dir = download_dir / "extracted" + zip_path = download_dir / "universial_fake_detection.zip" + shutil.rmtree(download_dir, ignore_errors=True) + download_dir.mkdir(parents=True, exist_ok=True) + + download_source = source_url or cls.source_url + file_id_match = re.search(r"/d/([^/]+)", download_source) or re.search(r"[?&]id=([^&]+)", download_source) + download_kwargs = {"id": file_id_match.group(1)} if file_id_match else {"url": download_source} + gdown.download( + output=str(zip_path), + quiet=False, + **download_kwargs, + ) + _safe_extract_zip(zip_path, extract_dir) + cls._flatten_extracted_tree(extract_dir, destination_dir) + + if not _has_images(destination_dir): + raise RuntimeError( + "Universal Fake Detection download did not contain images after " + "flattening diffusion_models into the test split." + ) + if not keep_downloads: + shutil.rmtree(download_dir, ignore_errors=True) + + @classmethod + def prepare_dataset( + cls, + root_dir: str | Path, + source_url: str | None = None, + force: bool = False, + keep_downloads: bool = False, + ) -> None: + """Download Universal Fake Detection as a test-only dataset under root_dir/test.""" + cls.prepare_split( + root_dir=root_dir, + split="test", + source_url=source_url, + force=force, + keep_downloads=keep_downloads, + ) + + def _load_data(self): + images, labels, folder_names = [], [], [] + for filepath in _iter_images(self.destination_dir): + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append(filepath.parent.parent.name) + return images, labels, folder_names + + def _get_label(self, filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name: + return 0 + if "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from filename: {filepath}") + + def _download_data(self, force: bool = False) -> None: + self.prepare_split( + root_dir=self.root_dir, + split=self.split, + source_url=self.source_url, + force=force, + ) + +class AIGIBenchDatasetModule: + def __init__(self, config): + self.config = config + train_list = list(config.DatasetConfig.train_transform_list) + eval_list = list(config.DatasetConfig.eval_transform_list) + self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + self.train_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "train", config.DatasetConfig.setting, self.train_transform, prefetch=prefetch) + self.val_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "val", config.DatasetConfig.setting, self.eval_transform, prefetch=prefetch) + self.test_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "test", config.DatasetConfig.setting, self.eval_transform, prefetch=prefetch) + + def train_dataloader(self): + return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def val_dataloader(self): + return DataLoader(self.val_dataset, batch_size=self.config.DatasetConfig.val_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class DiTFakeDatasetModule: + def __init__(self, config): + self.config = config + eval_list = list(config.DatasetConfig.eval_transform_list) + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + repo_id = config.DatasetConfig.get("repo_id", DiTFakeDataset.repo_id) + self.test_dataset = DiTFakeDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch, repo_id=repo_id) + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class FreqNetDetectionDatasetModule: + def __init__(self, config): + self.config = config + eval_list = list(config.DatasetConfig.eval_transform_list) + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + source_url = config.DatasetConfig.get("source_url", FreqNetDetectionDataset.google_drive_folder_url) + self.test_dataset = FreqNetDetectionDataset( + config.DatasetConfig.root_dir, + "test", + self.eval_transform, + prefetch=prefetch, + source_url=source_url, + ) + + def train_dataloader(self): + return None + + def val_dataloader(self): + return None + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class UniversialFakeDetectionDatasetModule: + def __init__(self, config): + self.config = config + eval_list = list(config.DatasetConfig.eval_transform_list) + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + source_url = config.DatasetConfig.get("source_url", UniversialFakeDetectionDataset.source_url) + self.test_dataset = UniversialFakeDetectionDataset( + config.DatasetConfig.root_dir, + "test", + self.eval_transform, + prefetch=prefetch, + source_url=source_url, + ) + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def train_dataloader(self): + return None + + def val_dataloader(self): + return None + + +class CoSpyDatasetModule: + def __init__(self, config): + self.config = config + eval_list = list(config.DatasetConfig.eval_transform_list) + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = getattr(config.DatasetConfig, "prefetch", False) + source_datasets = config.DatasetConfig.get("source_datasets", None) + generator_models = config.DatasetConfig.get("generator_models", None) + include_real = bool(config.DatasetConfig.get("include_real", True)) + repo_id = config.DatasetConfig.get("repo_id", CoSpyDataset.repo_id) + real_repo_id = config.DatasetConfig.get("real_repo_id", CoSpyDataset.real_repo_id) + real_filename = config.DatasetConfig.get("real_filename", CoSpyDataset.real_filename) + self.test_dataset = CoSpyDataset( + config.DatasetConfig.root_dir, + "test", + self.eval_transform, + prefetch=prefetch, + source_datasets=list(source_datasets) if source_datasets else None, + generator_models=list(generator_models) if generator_models else None, + include_real=include_real, + repo_id=repo_id, + real_repo_id=real_repo_id, + real_filename=real_filename, + ) + + def train_dataloader(self): + return None + + def val_dataloader(self): + return None + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class VideoFolderDataset(Dataset): + """Generic AI-generated-video benchmark backed by a folder of video files. + + Expected layout under ``root_dir/`` (mirrors ``CNNDetectionDataset``):: + + // + ├── / + │ ├── 0_real/*.mp4 (or real/) + │ └── 1_fake/*.mp4 (or fake/) + ├── / + │ └── ... + └── ... + + A flat ``//{0_real,1_fake}`` layout (no per-generator subdir) is also + supported. The label is read from the leaf directory name (``real`` -> 0, + ``fake``/``false`` -> 1) and the per-generator ``folder_name`` from the grandparent + directory, exactly like the image datasets, so the per-generator AP/AUROC metrics keep + working. ``__getitem__`` returns ``(frames[T, 3, H, W] float in [0, 1], label, folder_id_index)``. + + Downloading the (large) video benchmarks is out of scope: ``prefetch`` defaults to + ``False``. If ``hf_repo_id`` is configured the download stub fetches the split subtree + from the Hugging Face Hub; otherwise it raises ``NotImplementedError`` with guidance. + """ + + def __init__( + self, + root_dir: str | Path, + split: Literal["train", "val", "test"], + transform: AIGITransformations | None = None, + prefetch: bool = False, + num_frames: int = 16, + fps: float | None = None, + window_seconds: float = 2.0, + frame_size: int = 224, + hf_repo_id: str | None = None, + window_center: bool = False, + resize_interp: str = "bilinear", + ): + super().__init__() + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.num_frames = int(num_frames) + self.fps = fps if fps else None + self.window_seconds = float(window_seconds) + self.frame_size = int(frame_size) + self.hf_repo_id = hf_repo_id + self.window_center = bool(window_center) + self.resize_interp = str(resize_interp) + self.destination_dir = self.root_dir / self.split + + if prefetch and not _has_videos(self.destination_dir): + print(f"Downloading {self.split} video split into {self.destination_dir}...") + self._download_data() + + self.videos, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + @classmethod + def prepare_split( + cls, + root_dir: str | Path, + split: Literal["train", "val", "test"], + hf_repo_id: str | None = None, + force: bool = False, + ) -> None: + """Download one video split into ``root_dir/split`` (HF Hub only; stub otherwise).""" + root_dir = Path(root_dir) + destination_dir = root_dir / split + if not force and _has_videos(destination_dir): + return + if hf_repo_id: + from huggingface_hub import snapshot_download + + destination_dir.mkdir(parents=True, exist_ok=True) + snapshot_download( + repo_id=hf_repo_id, + repo_type="dataset", + local_dir=str(destination_dir), + allow_patterns=[f"{split}/**", f"{split}/*"], + ) + return + raise NotImplementedError( + f"No automatic downloader is configured for video split {destination_dir!s}. " + "Place the benchmark videos under '///{0_real,1_fake}/' " + "manually, or set 'hf_repo_id' in the dataset config to a Hugging Face dataset repo." + ) + + def _download_data(self, force: bool = False) -> None: + self.prepare_split( + root_dir=self.root_dir, + split=self.split, + hf_repo_id=self.hf_repo_id, + force=force, + ) + + def __len__(self): + return len(self.videos) + + def __getitem__(self, idx): + frames = sample_video_frames( + self.videos[idx], + num_frames=self.num_frames, + fps=self.fps, + window_seconds=self.window_seconds, + size=self.frame_size, + window_center=self.window_center, + interp=self.resize_interp, + ) + label = torch.tensor(self.labels[idx]) + if self.transform: + frames = self.transform(frames) + return frames, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name: + return 0 + if "fake" in name or "false" in name: + return 1 + raise ValueError(f"Cannot determine label from path: {filepath}") + + def _load_data(self): + videos, labels, folder_names = [], [], [] + split_dir = self.destination_dir + for filepath in _iter_videos(split_dir): + videos.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append( + filepath.parent.parent.name + if filepath.parent.parent != split_dir.parent + else self.split + ) + return videos, labels, folder_names + + +class VideoFolderDatasetModule: + def __init__(self, config): + self.config = config + cfg = config.DatasetConfig + train_list = list(cfg.get("train_transform_list", [])) + eval_list = list(cfg.get("eval_transform_list", [])) + self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + prefetch = bool(cfg.get("prefetch", False)) + common = dict( + num_frames=cfg.get("num_frames", 16), + fps=cfg.get("fps", None), + window_seconds=cfg.get("window_seconds", 2.0), + frame_size=cfg.get("frame_size", 224), + hf_repo_id=cfg.get("hf_repo_id", None), + window_center=cfg.get("window_center", False), + resize_interp=cfg.get("resize_interp", "bilinear"), + ) + self.train_dataset = VideoFolderDataset(cfg.root_dir, "train", self.train_transform, prefetch=prefetch, **common) + self.val_dataset = VideoFolderDataset(cfg.root_dir, "val", self.eval_transform, prefetch=prefetch, **common) + self.test_dataset = VideoFolderDataset(cfg.root_dir, "test", self.eval_transform, prefetch=prefetch, **common) + + def train_dataloader(self): + return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def val_dataloader(self): + return DataLoader(self.val_dataset, batch_size=self.config.DatasetConfig.val_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + def test_dataloader(self): + return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) + + +class AIGIDatasetModule: + def __init__(self, config): + self.delegate = SynthScopeModuleDict[config.DatasetConfig.name](config) + + def train_dataloader(self): + return self.delegate.train_dataloader() + + def val_dataloader(self): + return self.delegate.val_dataloader() + + def test_dataloader(self): + return self.delegate.test_dataloader() + +class EvalCrafterDataset(VideoFolderDataset): + """EvalCrafter T2V benchmark (Liu et al. 2024) — used by D3. + + All clips are AI-generated (no real videos), so every generator is placed under + ``/test/1_fake//*.mp4``. ``prepare_split`` snapshots the per-generator + ``*.tar.gz`` from the Hugging Face Hub and safe-extracts each into the ``1_fake`` tree. + """ + + hf_repo_id = "RaphaelLiu/EvalCrafter_T2V_Dataset" + + @classmethod + def prepare_split(cls, root_dir, split="test", hf_repo_id=None, force=False): + root_dir = Path(root_dir) + destination_dir = root_dir / split + if not force and _has_videos(destination_dir): + return + from huggingface_hub import snapshot_download + + download_dir = root_dir / "_downloads" + snapshot_download( + repo_id=hf_repo_id or cls.hf_repo_id, repo_type="dataset", + local_dir=str(download_dir), allow_patterns=["*.tar.gz"], + ) + fake_dir = destination_dir / "1_fake" + fake_dir.mkdir(parents=True, exist_ok=True) + for tar in sorted(download_dir.glob("*.tar.gz")): + _safe_extract_tar(tar, fake_dir) # -> 1_fake//*.mp4 + shutil.rmtree(download_dir, ignore_errors=True) + + def _download_data(self, force: bool = False) -> None: + self.prepare_split(self.root_dir, self.split, self.hf_repo_id, force) + + +class GenVidBenchDataset(VideoFolderDataset): + """GenVidBench (Ni et al. 2025) — used by ReStraV. + + Real sources (Vript, HD-VG-130M) -> ``/test/0_real//``; the 8 generators + (ms, pika, t2vz, vc2, cogvideo, mora, musev, svd) -> ``/test/1_fake//``. + ``prepare_split`` snapshots the per-source ``.rar``/multi-part ``.7z`` archives and + extracts them (needs the ``unrar`` binary; ``7z`` for the HD-VG archive). + """ + + hf_repo_id = "bitmind/GenVidBench" + REAL_SOURCES = {"vript", "hd_vg_130m"} + + @classmethod + def prepare_split(cls, root_dir, split="test", hf_repo_id=None, force=False): + root_dir = Path(root_dir) + destination_dir = root_dir / split + if not force and _has_videos(destination_dir): + return + from huggingface_hub import snapshot_download + + download_dir = root_dir / "_downloads" + snapshot_download(repo_id=hf_repo_id or cls.hf_repo_id, repo_type="dataset", local_dir=str(download_dir)) + for rar in sorted(download_dir.glob("**/*.rar")): + label = "0_real" if rar.stem.lower() in cls.REAL_SOURCES else "1_fake" + out = destination_dir / label + out.mkdir(parents=True, exist_ok=True) + cls._extract_rar(rar, out) + seven = sorted(download_dir.glob("**/hd_vg_130m.7z.001")) # multi-part 7z real source + if seven: + out = destination_dir / "0_real" + out.mkdir(parents=True, exist_ok=True) + cls._extract_7z(seven[0], out) + shutil.rmtree(download_dir, ignore_errors=True) + + @staticmethod + def _extract_rar(archive, out): + unrar = shutil.which("unrar") or "unrar" + subprocess.run([unrar, "x", "-o+", str(archive), str(out) + "/"], check=True, capture_output=True) + + @staticmethod + def _extract_7z(archive, out): + seven = shutil.which("7z") or "7z" + subprocess.run([seven, "x", "-y", f"-o{out}", str(archive)], check=True, capture_output=True) + for nested in Path(out).glob("*.7z"): + subprocess.run([seven, "x", "-y", f"-o{out}", str(nested)], check=True, capture_output=True) + nested.unlink(missing_ok=True) + + def _download_data(self, force: bool = False) -> None: + self.prepare_split(self.root_dir, self.split, self.hf_repo_id, force) + + +class _TestOnlyVideoModule: + """Shared loader for the eval-only video benchmarks (test split only).""" + + DATASET_CLASS = VideoFolderDataset + + def __init__(self, config): + self.config = config + cfg = config.DatasetConfig + eval_list = list(cfg.get("eval_transform_list", [])) + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_dataset = self.DATASET_CLASS( + cfg.root_dir, "test", self.eval_transform, prefetch=bool(cfg.get("prefetch", False)), + num_frames=cfg.get("num_frames", 16), fps=cfg.get("fps", None), + window_seconds=cfg.get("window_seconds", 2.0), frame_size=cfg.get("frame_size", 224), + hf_repo_id=cfg.get("hf_repo_id", self.DATASET_CLASS.hf_repo_id), + window_center=cfg.get("window_center", False), resize_interp=cfg.get("resize_interp", "bilinear"), + ) + + def train_dataloader(self): + return None + + def val_dataloader(self): + return None + + def test_dataloader(self): + cfg = self.config.DatasetConfig + return DataLoader(self.test_dataset, batch_size=cfg.test_batch_size, shuffle=False, + num_workers=cfg.num_workers, persistent_workers=cfg.persistent_workers, pin_memory=cfg.pin_memory) + + +class EvalCrafterDatasetModule(_TestOnlyVideoModule): + DATASET_CLASS = EvalCrafterDataset + + +class GenVidBenchDatasetModule(_TestOnlyVideoModule): + DATASET_CLASS = GenVidBenchDataset + + +# --------------------------------------------------------------------------- +# Eval-only cross-generator benchmark datasets (reproduction suite). Image benchmarks share a +# small test-only base + module (mirroring the video VideoFolderDataset / _TestOnlyVideoModule +# pattern); each subclass only declares its download source and restructures the download into +# the common ``/test//{0_real,1_fake}/`` layout used by the per-generator metrics. +# --------------------------------------------------------------------------- + +def _extract_archives_into(download_dir, staging_dir): + """Extract every .zip/.tar(.gz) under ``download_dir`` into ``staging_dir`` and return the dir + that holds the extracted generator tree (``staging_dir``; or ``download_dir`` if already flat).""" + download_dir, staging_dir = Path(download_dir), Path(staging_dir) + archives = [p for p in download_dir.rglob("*") + if p.is_file() and (p.suffix.lower() == ".zip" + or p.name.lower().endswith((".tar.gz", ".tgz", ".tar")))] + if not archives: + return download_dir + staging_dir.mkdir(parents=True, exist_ok=True) + for a in archives: + if a.suffix.lower() == ".zip": + _safe_extract_zip(a, staging_dir) + else: + _safe_extract_tar(a, staging_dir) + return staging_dir + + +def _restructure_image_pergen(src_root, dest_split, cap=None, rename=None): + """Lay a downloaded tree out as ``//{0_real,1_fake}/``. + + For each immediate subdirectory of ``src_root`` (one generator), images found under any + real-token dir (0_real/real/nature) become 0_real and fake-token dir (1_fake/fake/ai) become + 1_fake — searched recursively so per-class source-domain sublevels (e.g. DIRE's + ``//{0_real,1_fake}``) are merged. ``cap`` limits images/class; ``rename`` maps + the generator dir name (e.g. strip GenImage's ``_imagenet`` suffix).""" + src_root, dest_split = Path(src_root), Path(dest_split) + real_toks = ("0_real", "real", "reals", "nature", "natural") + fake_toks = ("1_fake", "fake", "fakes", "ai", "generated") + for gen_dir in sorted(p for p in src_root.iterdir() if p.is_dir()): + name = rename(gen_dir.name) if rename else gen_dir.name + for label, toks in (("0_real", real_toks), ("1_fake", fake_toks)): + imgs = [] + for tok in toks: + for sub in gen_dir.rglob(tok): + if sub.is_dir(): + imgs += [p for p in sub.rglob("*") + if p.is_file() and p.suffix.lower() in IMAGE_EXTENSIONS] + imgs = sorted(set(imgs)) + if cap: + imgs = imgs[:cap] + if not imgs: + continue + out = dest_split / name / label + out.mkdir(parents=True, exist_ok=True) + for i, p in enumerate(imgs): + tgt = out / f"{i:06d}_{p.name}" + if tgt.exists(): + continue + try: + os.link(p, tgt) + except OSError: + shutil.copy2(p, tgt) + + +def _restructure_video_pergen(src_root, dest_split, cap=None): + """Video analogue of ``_restructure_image_pergen`` → ``//{0_real,1_fake}/``.""" + src_root, dest_split = Path(src_root), Path(dest_split) + real_toks = ("0_real", "real", "reals") + fake_toks = ("1_fake", "fake", "fakes") + for gen_dir in sorted(p for p in src_root.iterdir() if p.is_dir()): + for label, toks in (("0_real", real_toks), ("1_fake", fake_toks)): + vids = [] + for tok in toks: + for sub in gen_dir.rglob(tok): + if sub.is_dir(): + vids += [p for p in sub.rglob("*") + if p.is_file() and p.suffix.lower() in VIDEO_EXTENSIONS] + vids = sorted(set(vids)) + if cap: + vids = vids[:cap] + if not vids: + continue + out = dest_split / gen_dir.name / label + out.mkdir(parents=True, exist_ok=True) + for i, p in enumerate(vids): + tgt = out / f"{i:06d}_{p.name}" + if tgt.exists(): + continue + try: + os.link(p, tgt) + except OSError: + shutil.copy2(p, tgt) + + +def _iter_images_follow(directory): + """Like ``_iter_images`` but follows symlinked directories (``os.walk`` with followlinks), + so generators whose ``0_real``/``1_fake`` are directory symlinks (e.g. GenImage's SD-v1.4 slice + reused from the single-pool dataset) are still discovered.""" + for dirpath, _dirnames, filenames in os.walk(Path(directory), followlinks=True): + for fn in filenames: + if Path(fn).suffix.lower() in IMAGE_EXTENSIONS: + yield Path(dirpath) / fn + + +class _TestOnlyImageDataset(Dataset): + """Base for eval-only image benchmarks laid out as ``/test//{0_real,1_fake}/``. + + Subclasses declare a download source and override ``prepare_split``; everything else (label + from the leaf dir name, per-generator ``folder_name`` from the grandparent dir, ``__getitem__`` + returning ``(image, label, folder_id)``) matches the other image datasets so the per-generator + AP/AUROC metrics keep working.""" + + def __init__(self, root_dir, split: Literal["test"] = "test", + transform: AIGITransformations | None = None, prefetch: bool = False): + super().__init__() + self.root_dir = Path(root_dir) + self.split = split + self.transform = transform + self.destination_dir = self.root_dir / self.split + if prefetch and not _has_images(self.destination_dir): + print(f"Downloading {type(self).__name__} {split} split...") + self._download_data() + self.images, self.labels, self.folder_names = self._load_data() + self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] + + @classmethod + def prepare_split(cls, root_dir, split: str = "test", force: bool = False) -> None: + raise NotImplementedError( + f"No downloader configured for {cls.__name__}; place the benchmark under " + f"'{Path(root_dir) / split}//{{0_real,1_fake}}/' manually.") + + def _download_data(self, force: bool = False) -> None: + self.prepare_split(self.root_dir, self.split, force=force) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + image = _decode_rgb(self.images[idx]) + label = torch.tensor(self.labels[idx]) + if self.transform: + image = self.transform(image) + return image, label, self.unique_folder_names.index(self.folder_names[idx]) + + @staticmethod + def _get_label(filepath: Path) -> int: + name = filepath.parent.name.lower() + if "real" in name or name in ("nature", "natural"): + return 0 + if "fake" in name or "false" in name or name == "ai": + return 1 + raise ValueError(f"Cannot determine label from path: {filepath}") + + def _load_data(self): + images, labels, folder_names = [], [], [] + for filepath in _iter_images_follow(self.destination_dir): + images.append(filepath) + labels.append(self._get_label(filepath)) + folder_names.append(filepath.parent.parent.name) + return images, labels, folder_names + + +class DiffusionForensicsDataset(_TestOnlyImageDataset): + """DiffusionForensics / DIRE diffusion test set (Wang et al. 2023) — used by NPR (Table 3) and + FatFormer's diffusion table. 8 subsets (adm, ddpm, iddpm, ldm, pndm, sdv1, sdv2, vqdiffusion), + each with its own paired reals (LSUN/ImageNet/CelebA-HQ).""" + gdrive_folder_id = "1tKsOU-6FDdstrrKLPYuZ7RpQwtOSHxUD" + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + import gdown + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + gdown.download_folder(id=cls.gdrive_folder_id, output=str(dl), quiet=False, use_cookies=False) + staging = _extract_archives_into(dl, root_dir / "_staging") + _restructure_image_pergen(staging, dest, cap=1000) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + +class Diffusion1kStepDataset(_TestOnlyImageDataset): + """Diffusion-1kStep self-synthesis test set — used by NPR (Table 5). 5 subsets (DALLE, ddpm, + guided-diffusion=ADM, improved-diffusion=IDDPM, midjourney).""" + gdrive_folder_id = "14f0vApTLiukiPvIHukHDzLujrvJpDpRq" + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + import gdown + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + gdown.download_folder(id=cls.gdrive_folder_id, output=str(dl), quiet=False, use_cookies=False) + staging = _extract_archives_into(dl, root_dir / "_staging") + _restructure_image_pergen(staging, dest, cap=1000) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + +class AIGCDetectBenchmarkDataset(_TestOnlyImageDataset): + """AIGCDetectBenchmark 16-generator test set — used by Co-Spy (ProGAN/AIGCDetect table) and + AIDE. Sourced from the ModelScope dataset ``aemilia/AIGCDetectionBenchmark`` (test_set.zip).""" + modelscope_url = ("https://modelscope.cn/api/v1/datasets/aemilia/AIGCDetectionBenchmark/" + "repo?Revision=master&FilePath=AIGCDetectionBenchMark%2Ftest_set.zip") + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + zip_path = dl / "test_set.zip" + subprocess.run(["curl", "-L", "--fail", "-C", "-", "-o", str(zip_path), cls.modelscope_url], + check=True) + staging = _extract_archives_into(dl, root_dir / "_staging") + _restructure_image_pergen(staging, dest, cap=1000) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + +class CommGen15Dataset(_TestOnlyImageDataset): + """CommGen15 — 15 commercial-generator test set introduced by PGC (Table 3). Sourced from the + Hugging Face dataset ``xiaoyuzhou68/CommGen15`` (CommGen15.zip; com_plat_v3//{0_real,1_fake}).""" + hf_repo_id = "xiaoyuzhou68/CommGen15" + hf_filename = "CommGen15.zip" + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + from huggingface_hub import hf_hub_download + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + z = hf_hub_download(repo_id=cls.hf_repo_id, filename=cls.hf_filename, + repo_type="dataset", local_dir=str(dl)) + staging = _extract_archives_into(Path(z).parent, root_dir / "_staging") + inner = next((p for p in staging.rglob("com_plat_v3") if p.is_dir()), staging) + _restructure_image_pergen(inner, dest, cap=1000) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + +class GenImageFullDataset(_TestOnlyImageDataset): + """Full 8-generator GenImage test set (Midjourney, SD-v1.4/1.5, ADM, GLIDE, Wukong, VQDM, + BigGAN) — used by AIDE/VIB-Net/SAFE/PGC/Co-Spy for the cross-generator GenImage table. (The + single-pool SD-v1.4 slice is the separate ``genimage`` dataset.) Sourced from the HF mirror + ``jzousz/GenImage`` (genimage_test.zip: test/_imagenet/{nature,ai}).""" + hf_repo_id = "jzousz/GenImage" + hf_filename = "genimage_test.zip" + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + from huggingface_hub import hf_hub_download + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + z = hf_hub_download(repo_id=cls.hf_repo_id, filename=cls.hf_filename, + repo_type="dataset", local_dir=str(dl)) + staging = _extract_archives_into(Path(z).parent, root_dir / "_staging") + inner = next((p for p in staging.rglob("test") if p.is_dir()), staging) + _restructure_image_pergen(inner, dest, cap=1500, rename=lambda n: n.replace("_imagenet", "")) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + +class SynthbusterDataset(_TestOnlyImageDataset): + """Synthbuster (Bammey, 2023) — 9 generators (DALL-E 2/3, Firefly, GLIDE, Midjourney-v5, SD 1.3/1.4/2/XL), + 1000 fakes each. Used by SPAI (Table 1) and other detectors as a high-quality diffusion test set. + Sourced from Zenodo record 10066460 (synthbuster.zip = /*.png, fakes only). Synthbuster ships + NO reals (the originals are RAISE-1k, which require a separate request); this loader pairs each + generator's fakes against a shared clean-real pool (GenImage nature photos) as a documented proxy — + the detector keys on generative artifacts, so proxy reals give a valid per-generator AP/AUROC.""" + zenodo_record = "10066460" + zenodo_file = "synthbuster.zip" + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False, real_pool=None, cap=1000): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + import urllib.request + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + zpath = dl / cls.zenodo_file + if not zpath.exists(): + url = f"https://zenodo.org/records/{cls.zenodo_record}/files/{cls.zenodo_file}?download=1" + print(f"Downloading Synthbuster from {url} ...") + urllib.request.urlretrieve(url, zpath) + staging = _extract_archives_into(dl, root_dir / "_staging") + inner = next((p for p in staging.rglob("synthbuster") if p.is_dir()), staging) + # proxy reals: a clean real-photo pool (GenImage nature by default) + pool = real_pool or "/hddstore/ssalehi/SynthScope/GenImage/test/real" + reals = sorted(p for p in Path(pool).glob("*") if p.suffix.lower() in IMAGE_EXTENSIONS)[:cap] + for gen_dir in sorted(p for p in inner.iterdir() if p.is_dir()): + fakes = sorted(p for p in gen_dir.glob("*") if p.suffix.lower() in IMAGE_EXTENSIONS)[:cap] + for label, imgs in (("1_fake", fakes), ("0_real", reals)): + out = dest / gen_dir.name / label; out.mkdir(parents=True, exist_ok=True) + for i, p in enumerate(imgs): + tgt = out / f"{i:05d}_{p.name}" + if not tgt.exists(): + try: os.link(p, tgt) + except OSError: shutil.copy2(p, tgt) + shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + +class _TestOnlyImageModule: + """Shared loader for the eval-only image benchmarks (test split only).""" + DATASET_CLASS = _TestOnlyImageDataset + + def __init__(self, config): + self.config = config + cfg = config.DatasetConfig + eval_list = list(cfg.get("eval_transform_list", [])) + self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_dataset = self.DATASET_CLASS( + cfg.root_dir, "test", self.eval_transform, prefetch=bool(cfg.get("prefetch", False))) + + def train_dataloader(self): + return None + + def val_dataloader(self): + return None + + def test_dataloader(self): + cfg = self.config.DatasetConfig + return DataLoader(self.test_dataset, batch_size=cfg.test_batch_size, shuffle=False, + num_workers=cfg.num_workers, persistent_workers=cfg.persistent_workers, + pin_memory=cfg.pin_memory) + + +class DiffusionForensicsDatasetModule(_TestOnlyImageModule): + DATASET_CLASS = DiffusionForensicsDataset + + +class Diffusion1kStepDatasetModule(_TestOnlyImageModule): + DATASET_CLASS = Diffusion1kStepDataset + + +class AIGCDetectBenchmarkDatasetModule(_TestOnlyImageModule): + DATASET_CLASS = AIGCDetectBenchmarkDataset + + +class CommGen15DatasetModule(_TestOnlyImageModule): + DATASET_CLASS = CommGen15Dataset + + +class GenImageFullDatasetModule(_TestOnlyImageModule): + DATASET_CLASS = GenImageFullDataset + + +class SynthbusterDatasetModule(_TestOnlyImageModule): + DATASET_CLASS = SynthbusterDataset + + +class VideoFeedbackDataset(VideoFolderDataset): + """VideoFeedback (TIGER-Lab) — used by STALL. 11 generators vs DiDeMo+Panda70M real, laid out + as ``/test//{0_real,1_fake}/*.mp4``.""" + hf_repo_id = "TIGER-Lab/VideoFeedback" + + @classmethod + def prepare_split(cls, root_dir, split="test", hf_repo_id=None, force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_videos(dest): + return + from huggingface_hub import snapshot_download + dl = root_dir / "_downloads" + snapshot_download(repo_id=hf_repo_id or cls.hf_repo_id, repo_type="dataset", local_dir=str(dl)) + staging = _extract_archives_into(dl, root_dir / "_staging") + _restructure_video_pergen(staging, dest, cap=150) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + def _download_data(self, force=False): + self.prepare_split(self.root_dir, self.split, self.hf_repo_id, force) + + +class ComGenVidDataset(VideoFolderDataset): + """ComGenVid (STALL's new benchmark) — Sora + VEO3 fakes vs MSVD real, laid out as + ``/test//{0_real,1_fake}/*.mp4``. The HF repo ``OmerXYZ/comgenvid`` also ships + precomputed embeddings for STALL's official embedding-based eval.""" + hf_repo_id = "OmerXYZ/comgenvid" + + @classmethod + def prepare_split(cls, root_dir, split="test", hf_repo_id=None, force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_videos(dest): + return + from huggingface_hub import snapshot_download + dl = root_dir / "_downloads" + snapshot_download(repo_id=hf_repo_id or cls.hf_repo_id, repo_type="dataset", + local_dir=str(dl), allow_patterns=["*.mp4", "**/*.mp4"]) + staging = _extract_archives_into(dl, root_dir / "_staging") + _restructure_video_pergen(staging, dest, cap=150) + shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(root_dir / "_staging", ignore_errors=True) + + def _download_data(self, force=False): + self.prepare_split(self.root_dir, self.split, self.hf_repo_id, force) + + +class VideoFeedbackDatasetModule(_TestOnlyVideoModule): + DATASET_CLASS = VideoFeedbackDataset + + +class ComGenVidDatasetModule(_TestOnlyVideoModule): + DATASET_CLASS = ComGenVidDataset + + +SynthScopeModuleDict = { + "aigibench": AIGIBenchDatasetModule, + "cnndetection": CNNDetectionDatasetModule, + "cospy": CoSpyDatasetModule, + "ditfake": DiTFakeDatasetModule, + "freqnet": FreqNetDetectionDatasetModule, + "freqnetdetection": FreqNetDetectionDatasetModule, + "genimage": GenImageDatasetModule, + "universialfakedetection": UniversialFakeDetectionDatasetModule, + # Cross-generator image benchmarks added for the reproduction suite (test-only). + "genimage_full": GenImageFullDatasetModule, + "synthbuster": SynthbusterDatasetModule, + "diffusionforensics": DiffusionForensicsDatasetModule, + "diffusion1kstep": Diffusion1kStepDatasetModule, + "aigcdetect": AIGCDetectBenchmarkDatasetModule, + "commgen15": CommGen15DatasetModule, + # Video benchmarks. EvalCrafter/GenVidBench/VideoFeedback/ComGenVid have dedicated classes that + # download + extract their HF archives into the real/fake layout; the rest use the generic + # VideoFolderDatasetModule. + "genvideo": VideoFolderDatasetModule, + "videophy": VideoFolderDatasetModule, + "evalcrafter": EvalCrafterDatasetModule, + "vidprom": VideoFolderDatasetModule, + "genvidbench": GenVidBenchDatasetModule, + "physics_iq": VideoFolderDatasetModule, + "videofeedback": VideoFeedbackDatasetModule, + "comgenvid": ComGenVidDatasetModule, +} + + +if __name__ == "__main__": + config = compose_config() + dataset_module = SynthScopeModuleDict[config.DatasetConfig.name](config) + train_dataloader = dataset_module.train_dataloader() + val_dataloader = dataset_module.val_dataloader() + test_dataloader = dataset_module.test_dataloader() + for batch in train_dataloader: + print(batch) + break + for batch in val_dataloader: + print(batch) + break + for batch in test_dataloader: + print(batch) + plt.imshow(batch[0].permute(1, 2, 0)) + plt.savefig(fname='test_image.png') + break diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c21e94f1eaf8ddc135697141223835026a136d05 --- /dev/null +++ b/demo/README.md @@ -0,0 +1,42 @@ +--- +title: SynthScope AI-Generated Media Detection +emoji: 🔎 +colorFrom: indigo +colorTo: red +sdk: gradio +sdk_version: "4.44.1" +app_file: app.py +pinned: false +license: apache-2.0 +--- + +# SynthScope demo (Gradio / Hugging Face Spaces) + +Upload an image or video, pick a detector, and get the probability it is AI-generated. Built on +the [`synthscope`](https://github.com/SMSD75/SynthScope) Python API (`synthscope.load(key).predict(file)`). + +## Run locally + +```bash +pip install -e ".[demo]" # from the SynthScope repo root +python demo/app.py # serves on http://localhost:7860 +``` + +## Deploy as a Hugging Face Space + +1. Create a new **Gradio** Space. +2. Add these three files at the Space **root**: `app.py`, `requirements.txt`, and this `README.md` + (the YAML header above configures the Space). +3. Set a Space **variable** `SYNTHSCOPE_HUB_REPO=/synthscope-checkpoints` so detectors + auto-download their weights from your [model zoo](../README.md#model-zoo-hugging-face-hub). For a + **private** zoo, also add an `HF_TOKEN` Space **secret**. +4. Choose hardware: the large backbones (ConvNeXt-XXL, SigLIP, CLIP-L) want a **GPU**; CPU works but + is slow. + +## Notes + +- Detectors needing **gated** backbones (DINOv3 → `dinov3`, `stall`) are hidden in this demo. +- Official-weights detectors (`fat_former`, `aide`, `npr`) only appear if their weights are in the + configured zoo (they are license-restricted and not published by default — see `LICENSES.md`). +- A single score is a demo, not a verdict — see the repo's per-generator reproduction table for + benchmarked accuracy. diff --git a/demo/app.py b/demo/app.py new file mode 100644 index 0000000000000000000000000000000000000000..2ac082990d53ae65eaa46f8c0f8beeee50bb1029 --- /dev/null +++ b/demo/app.py @@ -0,0 +1,77 @@ +"""SynthScope Gradio demo — runs any detector on an uploaded image or video. + +Deployable as a Hugging Face Space (see demo/README.md) or locally: + + pip install -e ".[demo]" + python demo/app.py + +On a Space (or any machine without local weights), detectors auto-download from the model zoo; +set the ``SYNTHSCOPE_HUB_REPO`` env var to your checkpoints repo. +""" +from __future__ import annotations + +import synthscope + +# Detectors that need gated backbones we can't host on a public demo (DINOv3 weights). +_GATED = {"dinov3", "stall"} + + +def _model_choices(): + """Dropdown choices as (label, key), available/runnable detectors first.""" + models = [m for m in synthscope.list_models() if m["key"] not in _GATED] + models.sort(key=lambda m: (not m["available"], m["modality"], m["key"])) + return [(f"{m['name']} · {m['paper']} ({m['modality']})", m["key"]) for m in models] + + +def run_detection(file_path, model_key): + """Score ``file_path`` with detector ``model_key``; return (label_dict, details_dict).""" + if not file_path: + return {}, {"error": "Upload an image or video first."} + if not model_key: + return {}, {"error": "Pick a detector."} + try: + det = synthscope.load(model_key) + res = det.predict(file_path) + except FileNotFoundError: + return {}, {"error": f"No weights available for '{model_key}' " + "(not in the model zoo, or gated). Try another detector."} + except Exception as exc: # surface decode/runtime errors to the UI + return {}, {"error": f"{type(exc).__name__}: {exc}"} + label = { + "AI-generated (fake)": res["fake_probability"], + "Real": res["real_probability"], + } + return label, res + + +def build_demo(): + import gradio as gr + + choices = _model_choices() + default = choices[0][1] if choices else None + with gr.Blocks(title="SynthScope — AI-generated media detection") as demo: + gr.Markdown( + "# SynthScope — AI-generated image & video detection\n" + "Upload an **image or video** and pick a detector to estimate the probability that it " + "is **AI-generated**. Each detector is a re-implementation benchmarked in " + "[SynthScope](https://github.com/SMSD75/SynthScope)." + ) + with gr.Row(): + with gr.Column(): + file_in = gr.File(label="Image or video", type="filepath", + file_types=["image", "video"]) + model_in = gr.Dropdown(choices=choices, value=default, label="Detector") + btn = gr.Button("Detect", variant="primary") + with gr.Column(): + label_out = gr.Label(label="Prediction", num_top_classes=2) + json_out = gr.JSON(label="Details") + btn.click(run_detection, inputs=[file_in, model_in], outputs=[label_out, json_out]) + gr.Markdown( + "_Per-generator generalization is the hard part of this task; a single score is a demo, " + "not a verdict. See the repo's reproduction table for benchmarked accuracy._" + ) + return demo + + +if __name__ == "__main__": + build_demo().launch(server_name="0.0.0.0") diff --git a/demo/requirements.txt b/demo/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..cfe398c10a37270fc39d95f09d0d1b56e428b799 --- /dev/null +++ b/demo/requirements.txt @@ -0,0 +1,6 @@ +# Hugging Face Space dependencies. Spaces installs these automatically. +# SynthScope itself (pulls torch/torchvision/transformers/etc. per pyproject): +synthscope @ git+https://github.com/SMSD75/SynthScope.git +gradio>=4 +# A few detectors (clip, fat_former) also need OpenAI CLIP: +clip @ git+https://github.com/openai/CLIP.git diff --git a/deploy/hf-space/Dockerfile b/deploy/hf-space/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..51451c10b5f8b8d2d7d2144cb3a9c92b19e6bf01 --- /dev/null +++ b/deploy/hf-space/Dockerfile @@ -0,0 +1,21 @@ +# SynthScope web page as a Hugging Face Docker Space (CPU hardware, e.g. "CPU Upgrade": 8 vCPU, 32 GB). +# Build context = the repository root (push_space.py copies this file to the root of the Space repo). +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends git ffmpeg libgl1 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* +# Spaces run the container as user 1000 +RUN useradd -m -u 1000 user +USER user +ENV HOME=/home/user PATH=/home/user/.local/bin:$PATH PYTHONUNBUFFERED=1 \ + SYNTHSCOPE_CKPT_DIR=/home/user/ckpts SYNTHSCOPE_CACHE_DIR=/home/user/cache HF_HOME=/home/user/cache/hf \ + SYNTHSCOPE_MAX_UPLOAD_MB=20 SYNTHSCOPE_MAX_RESIDENT_MODELS=2 XFORMERS_DISABLED=1 +WORKDIR /home/user/app + +# CPU-only PyTorch first (much smaller than the CUDA build), then SynthScope with the web extras +RUN pip install --no-cache-dir --user torch torchvision --index-url https://download.pytorch.org/whl/cpu +COPY --chown=user . . +RUN pip install --no-cache-dir --user -e ".[web]" "git+https://github.com/openai/CLIP.git" + +EXPOSE 7860 +CMD ["bash", "deploy/hf-space/start.sh"] diff --git a/deploy/hf-space/SPACE_README.md b/deploy/hf-space/SPACE_README.md new file mode 100644 index 0000000000000000000000000000000000000000..33bd2a221023e51204635669e86df388413ca196 --- /dev/null +++ b/deploy/hf-space/SPACE_README.md @@ -0,0 +1,22 @@ +--- +title: SynthScope +emoji: 🔎 +colorFrom: indigo +colorTo: blue +sdk: docker +app_port: 7860 +pinned: false +license: apache-2.0 +short_description: Is this image AI-generated? Compare published detectors. +--- + +# SynthScope + +Check whether an image or video is AI-generated with published detectors, see what each detector looks at, and compare how +they hold up against today's image generators. This Space runs on CPU: a detector takes a few seconds per image. + +Code, benchmarks and documentation: **https://github.com/SMSD75/SynthScope** + +Each detector and its weights keep the license chosen by their authors (see the repository's `LICENSES.md`); this demo is +free and non-commercial. If you are an author and would like your method removed or credited differently, please open an +issue on GitHub. diff --git a/deploy/hf-space/fetch_weights.py b/deploy/hf-space/fetch_weights.py new file mode 100644 index 0000000000000000000000000000000000000000..71f60d9a11511df74c7ba039d90dff8d20dcb4d6 --- /dev/null +++ b/deploy/hf-space/fetch_weights.py @@ -0,0 +1,45 @@ +"""Download the detector weights the public Space may serve, from a Hugging Face model repo. + +Configure the Space with two settings (Space -> Settings -> Variables and secrets): + SYNTHSCOPE_HUB_REPO model repo holding .pth files (it can be private) e.g. SMSD75/synthscope-checkpoints + HF_TOKEN secret; a read token for that repo (only needed if the repo is private) +SYNTHSCOPE_SPACE_DETECTORS (comma list) overrides which detectors are fetched. Missing files are skipped, so a detector +without weights simply shows as unavailable on the page. Training-free detectors (D3) need no file. +""" +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +# Detectors whose upstream licenses allow a free public demo. NPR's repository has no license: we asked the authors +# (2026-10-04) and drop it if they decline. +DEFAULT = ["clip", "cnn", "npr", "safe", "aide", "co_spy", "fat_former", "pgc", "cure", "vib_net", "dinov2", "adapted_dino", + "restrav"] + + +def main() -> None: + repo = os.environ.get("SYNTHSCOPE_HUB_REPO") + if not repo: + print("[weights] SYNTHSCOPE_HUB_REPO is not set: starting without detector weights", flush=True) + return + keys = [k for k in os.environ.get("SYNTHSCOPE_SPACE_DETECTORS", ",".join(DEFAULT)).split(",") if k] + from synthscope import hub + token = os.environ.get("HF_TOKEN") or None + ckpt_dir = Path(os.environ.get("SYNTHSCOPE_CKPT_DIR", "~/ckpts")).expanduser() + # D3 is training-free (no .pth) but turns its raw score into a probability with a small calibration file + if not (ckpt_dir / "d3_calibration.json").exists(): + hub._maybe_download_aux("d3", repo, token) + for key in keys: + if (ckpt_dir / f"{key}.pth").exists(): + continue + try: + hub.download(key, repo_id=repo, token=token) + print(f"[weights] {key}: ok", flush=True) + except Exception as e: # keep starting: the page marks the detector as unavailable + print(f"[weights] {key}: skipped ({type(e).__name__})", flush=True) + + +if __name__ == "__main__": + main() diff --git a/deploy/hf-space/push_space.py b/deploy/hf-space/push_space.py new file mode 100644 index 0000000000000000000000000000000000000000..e27eba67d52ec0a361aec84b8516afd4f7971d38 --- /dev/null +++ b/deploy/hf-space/push_space.py @@ -0,0 +1,58 @@ +"""Assemble the Hugging Face Space from this repository and (only with --confirm) upload it. + + python deploy/hf-space/push_space.py --space SMSD75/SynthScope # dry run: list what would be uploaded + python deploy/hf-space/push_space.py --space SMSD75/SynthScope --confirm # create/update the Space + +The Space gets the tracked repository files (minus job logs), with this folder's Dockerfile at the root and SPACE_README.md as +its README.md (the Space card). Needs a Hugging Face token with write access (HF_TOKEN or `huggingface-cli login`). +After the first upload, set SYNTHSCOPE_HUB_REPO (and HF_TOKEN as a secret, if the weights repo is private) in the Space settings, +and pick the hardware (e.g. "CPU upgrade"). +""" +import argparse +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +HERE = Path(__file__).resolve().parent +EXCLUDE = ("repro/logs/",) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--space", required=True, help="owner/name of the Space") + ap.add_argument("--private", action="store_true", help="create the Space as private (it can be made public later)") + ap.add_argument("--hardware", default="cpu-upgrade", + help="Space hardware when creating it (Docker Spaces on free 'cpu-basic' need a PRO subscription)") + ap.add_argument("--sleep", type=int, default=3600, help="seconds without visitors before the Space sleeps (paid hardware)") + ap.add_argument("--confirm", action="store_true", help="actually upload (default: dry run)") + a = ap.parse_args() + + files = [f for f in subprocess.check_output(["git", "ls-files"], cwd=ROOT, text=True).splitlines() + if f and not f.startswith(EXCLUDE) and (ROOT / f).is_file()] + size = sum((ROOT / f).stat().st_size for f in files) + print(f"{len(files)} files, {size / 1e6:.1f} MB from the repository, plus Dockerfile and the Space card") + if not a.confirm: + print("dry run: nothing uploaded (add --confirm to upload)") + return + + from huggingface_hub import HfApi + api = HfApi(token=os.environ.get("HF_TOKEN")) + api.create_repo(a.space, repo_type="space", space_sdk="docker", private=a.private, exist_ok=True, + space_hardware=a.hardware, space_sleep_time=a.sleep) + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + for f in files: + (tmp / f).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / f, tmp / f) + shutil.copy2(HERE / "Dockerfile", tmp / "Dockerfile") + shutil.copy2(HERE / "SPACE_README.md", tmp / "README.md") + api.upload_folder(folder_path=str(tmp), repo_id=a.space, repo_type="space", + commit_message="Update SynthScope Space from the GitHub repository") + print(f"uploaded: https://huggingface.co/spaces/{a.space}") + + +if __name__ == "__main__": + main() diff --git a/deploy/hf-space/start.sh b/deploy/hf-space/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..31cce10d8ebea6ab9a8547b4a37aa3b28047b794 --- /dev/null +++ b/deploy/hf-space/start.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Space entry point: fetch the approved detector weights, then serve the page on the port Spaces expect (7860). +set -e +python deploy/hf-space/fetch_weights.py +# Openly licensed datasets for the "Browse datasets" tab, downloaded in the background (the tab appears once one is ready). +# SYNTHSCOPE_SPACE_DATASETS="" turns this off. +export SYNTHSCOPE_DATA_DIR=${SYNTHSCOPE_DATA_DIR:-$HOME/data} +( + for spec in ${SYNTHSCOPE_SPACE_DATASETS-image:commgen15:CommGen15 video:evalcrafter:EvalCrafter}; do + IFS=: read -r modality name folder <<< "$spec" + python benchmark/download_dataset.py "$name" --modality "$modality" --root "$SYNTHSCOPE_DATA_DIR/$folder" \ + > "$HOME/download_$name.log" 2>&1 && echo "[data] $name ready" || echo "[data] $name failed (see $HOME/download_$name.log)" + done +) & +exec python -m uvicorn interface.backend.app:app --host 0.0.0.0 --port 7860 diff --git a/docs/ADD_DETECTOR.md b/docs/ADD_DETECTOR.md new file mode 100644 index 0000000000000000000000000000000000000000..fb34d6cf970a1ca4349d8b7edfd912cf9167c7fa --- /dev/null +++ b/docs/ADD_DETECTOR.md @@ -0,0 +1,52 @@ +# Adding a detector to SynthScope (the repo rules) + +This is the checklist the **What's-Trending daily agent** (and humans) follow to incorporate a new +AI-generated-content detector cleanly and verify that SynthScope reproduces the paper's results. Keep the +code style and structure consistent with the existing detectors (PGC, SAFE, Co-Spy, AIDE, …). + +## When to integrate +Only integrate a paper that satisfies **both** trending rules: (1) a solid public GitHub repo, and +(2) solid results vs previous methods. If the architecture is unclear, weights aren't released, or +the benchmark data can't be obtained, **do not** commit broken code — record an "integration pending: +" note in the paper's `trending.json` entry and stop. + +## Files to add (one detector = key ``, e.g. `pgc`) +1. **`models///.py`** — `class SynthScopeModel(nn.Module)`: + - `__init__(self, config: DictConfig)` — build the architecture; respect `config.ModelConfig`. + - `forward(self, images) -> logits` — raw logits (NOT probabilities). Do model-specific + preprocessing (e.g. residual/DWT/DCT) **inside** forward where the official model does. + - `compute_loss(self, batch) -> {"loss", "outputs"}` — used by training. + - Imports + helper classes at module top-level (repo rule: no nested defs/imports). + - Add `models///__init__.py` and a `README.md` (see structure below). +2. **`conf/model/image/.yaml`** (or `video/`): `defaults: [/model/default, _self_]`, then `model_type: ` + plus any model knobs. Add `conf/dataset/...` / `conf/trainer/...` only if a new recipe is needed. +3. **Dispatch** — add a branch in `trainer.py` `_build_model` (the `if model_type == "": return SynthScopeModel(config)` chain, ~line 69+). The interface lazy-imports by `model_type` in `interface/backend/engine.py` — mirror it if needed. +4. **Interface registry** — `interface/backend/registry.py` `_add(ModelSpec(...))`: + - `key`, `name`, `paper`, `modality` (`image`|`video`), `overrides=["model@ModelConfig=image/"]`, + - `predict_kind`: `ce` (logits→`softmax[:,1]`), `bce` (`sigmoid`), `bce_npr`, `vib`, or `video`, + - `checkpoint=".pth"`, `ckpt_kind`: `synthscope` (our trained weights, strict load) **or** `official_` + a `remap=remap_` function (rename official keys → our module names) for the authors' released weights, + - `transform`: the eval preprocessing — match the **official** eval exactly (resize vs crop, norm vs none). Add a `tf_*` helper if needed. (Getting this wrong silently tanks results — see SAFE: a resize destroyed the high-freq signal.) + - `weights`: `"official"` | `"synthscope_trained"` | `"training_free"` (shown in the web UI badge). +5. **Per-model `README.md`** — mirror `models/image/pgc/README.md`: short method explanation + official-repo/paper links, the pipeline, "SynthScope adaptations", and a results table in the **Paper / SynthScope_official / SynthScope_train** schema. + +## Verify reproduction (required) +Run the detector on the **paper's own benchmark(s)** and compare to the paper's reported numbers: +``` +python benchmark/bench_eval.py --model --benchmark {forensynths|ufd|pergen|pool} --data \ + --ckpt --bs 32 --device cuda --tag _repro +``` +- Loading **official weights** with `--ckpt` checks architecture fidelity (should match the paper). +- Mismatched `load_state_dict` keys → write a `remap_`; near-chance/all-fake → check the transform + (preprocessing) and the head-name mapping first. +- Record per-generator AP/AUROC; update `repro/AGGREGATED_BENCHMARK.md`, `repro/RESULTS_SUMMARY.md`, + the repo README table, and the per-model README — all in the Paper / SynthScope_official / SynthScope_train schema. + +## Git workflow (keep main clean) +- Do the integration on a branch: `git checkout -b trending/`. Commit the model + conf + registry + + README + repro results. **Push the branch and surface it for review — do NOT auto-merge to `main`.** +- End commit messages with the repo's `Co-Authored-By` trailer. +- Never commit weights (`*.pth`), `.env`, or caches (already gitignored). + +## Tests +Add a smoke test under `tests/` like `tests/test__smoke.py` (mirror `tests/test_aide_smoke.py`): +build the model, run a dummy `forward`, assert output shape + finite logits. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 0000000000000000000000000000000000000000..c0ef939aa08e678f232da4fa91303938428990af --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,212 @@ +# Setting up and running the benchmarks + +SynthScope has **three kinds of benchmark**. They answer different questions, use different commands, and feed different +places in the web page. Pick the one you need: + +| I want to... | Run | Result appears in | +| --- | --- | --- | +| **Check a detector against its paper** (does our port reproduce the published numbers?) | [`benchmark/bench_eval.py`](../benchmark/bench_eval.py) (images), [`bench_eval_video.py`](../benchmark/bench_eval_video.py) (video) | the page's **Leaderboard** tab, the tables in each model's `README.md` | +| **See how detectors hold up against a particular image generator** (FLUX, Qwen, Nano Banana, ...) | [`benchmark/generator_bench.py`](../benchmark/generator_bench.py) | the page's **Generator benchmarks** tab | +| **Submit a number to the standardized, versioned leaderboard** | [`benchmark/evaluate.py`](../benchmark/evaluate.py) | [`LEADERBOARD.md`](../LEADERBOARD.md), see [`benchmark/README.md`](../benchmark/README.md) | + +Everything below assumes you installed SynthScope (`pip install -e .`, see the [README](../README.md#-install)) and, for the +GPU steps, have a CUDA GPU. Detectors run on CPU too, but the larger ones (AIDE, SPAI, CuRe) are slow there. + +--- + +## 1. Get the detector weights + +A detector needs its checkpoint before it can be scored. + +```bash +synthscope list # the AVAIL column shows which detectors have weights on this machine +export SYNTHSCOPE_CKPT_DIR=/path/to/checkpoints # where .pth files live (default ~/.cache/synthscope/checkpoints) +``` + +- **Official weights** (NPR, FatFormer, AIDE, SPAI, PGC, CuRe, ...) come from the authors; each model's + `models///README.md` says where to download them and what to name the file. Their licenses are your + responsibility ([`LICENSES.md`](../LICENSES.md)). +- **SynthScope-trained weights**: train with the recipe in the README's *Train your own* section, or fetch released ones with + `synthscope pull --model ` once a model-zoo repo is configured (`SYNTHSCOPE_HUB_REPO`). +- Some detectors need extra backbones: SPAI needs the MFM backbone (`SYNTHSCOPE_MFM_CKPT`), CuRe the Perception-Encoder + base (`SYNTHSCOPE_PE_CORE_CKPT`, or it downloads `facebook/PE-Core-L14-336`), the DINOv3 probe and STALL the gated DINOv3 + weights (`DINO_V3_WEIGHTS`, `DINO_V3_WEIGHTS_DIR`). + +--- + +## 2. Datasets: they download and organize themselves + +You do not have to find benchmark data or rearrange folders. Each supported dataset has a config in +[`conf/dataset/`](../conf/dataset/) that records its official source, and the dataset's loader **downloads it, unpacks it safely +(zip/tar extraction guards against path tricks) and restructures it into the one layout every SynthScope tool reads**: + +``` +/ + /0_real/*.png /1_fake/*.png + /0_real/*.png /1_fake/*.png +``` + +The evaluators, the training loop and the web page's *Browse datasets* tab all rely on that layout, which is why no per-dataset +glue code is needed. We check this end to end: downloading CommGen15 (about 26,000 images, 4 GB) with the command below took +about two minutes, produced `test//{0_real,1_fake}` for 15 generators, and `bench_eval.py` scored it straight away. + +```bash +python benchmark/download_dataset.py --list # every dataset and where it comes from +python benchmark/bench_eval.py --model npr --dataset commgen15 --root /data/CommGen15 # download if missing, then evaluate +python benchmark/download_dataset.py commgen15 --root /data/CommGen15 # only download + lay out +``` + +`bench_eval.py --dataset NAME` is the one-step version: it downloads the dataset when no images are under `--root` (or the +`root_dir` in the dataset's yaml), finds the folder to evaluate (`/test`, `/test_full` or the root itself), picks the +grouping profile from the layout (UnivFD and ForenSynths have their own), and runs the evaluation. `--no-download` makes it fail +instead of fetching, and `--dry-run` only shows what it resolved. An incomplete download is not detected: delete the folder to +fetch it again. + +Training works the same way: a dataset config with `prefetch: true` fetches its data the first time it is used, into +`DatasetConfig.root_dir`. Downloads can be large (CommGen15 about 10 GB on the Hub, DiTFake about 22 GB, AIGIBench about +220 GB), so check your disk first. Re-running skips data that is already in place. + +Datasets with an **Auto** download below are fetched by the loader. A few video sets (GenVideo, VidProM, VideoPhy, Physics-IQ) +have no automatic source: get them from their official sites and place them in the same layout. + +| Dataset (config name) | What it is | Source (download) | +| --- | --- | --- | +| `cnndetection` | ForenSynths: ProGAN training set and the GAN test sets | Google Drive (`source_url` in the yaml) | +| `universialfakedetection` | UnivFD test sets (LDM, Glide, DALL-E, guided) | Google Drive | +| `genimage`, `genimage_full` | GenImage, 8 generators | Hugging Face `jzousz/GenImage` (Google Drive is quota-limited) | +| `aigibench` | AIGIBench | Hugging Face `HorizonTEL/AIGIBench` | +| `aigcdetect` | AIGCDetectBenchmark, 16 generators | ModelScope `aemilia/AIGCDetectionBenchmark` | +| `cospy` | Co-Spy-Bench | Hugging Face `ruojiruoli/Co-Spy-Bench` | +| `synthbuster` | Synthbuster | Zenodo record `10066460` | +| `ditfake`, `commgen15` | DiTFake, CommGen15 | Hugging Face (`lioooox/DiTFake`, `xiaoyuzhou68/CommGen15`) | +| `diffusionforensics`, `diffusion1kstep`, `freqnetdetection` | DiffusionForensics, Diffusion-1kStep, FreqNet sets | see the yamls | +| Video: `genvideo`, `genvidbench`, `evalcrafter`, `videofeedback`, `vidprom`, `comgenvid`, `physics_iq`, `videophy` | the video benchmarks | Hugging Face where a repo is set; otherwise place the files by hand | + +**If a download fails.** Hosts sometimes rate-limit (Google Drive) or ask you to accept terms first. Fetch the data by hand from the +source in the table and unpack it so it matches the layout above (`real/`+`fake/` and `nature/`+`ai/` folders are also +understood; see the docstring of [`bench_eval.py`](../benchmark/bench_eval.py)). Datasets are large, so keep them on fast +local disk. + +--- + +## 3. Reproduce a paper's numbers + +```bash +python benchmark/bench_eval.py --model clip --dataset cnndetection --root /data/CNNDetection # download if needed + evaluate +# the explicit form, for data you arranged yourself: +python benchmark/bench_eval.py --model clip --benchmark forensynths --data /data/CNNDetection/test --tag clip_forensynths +``` + +With `--dataset` the grouping below is chosen for you; with `--data` you pick `--benchmark` yourself: + +| `--benchmark` | Layout it expects | Typical dataset | +| --- | --- | --- | +| `forensynths` | `//.../{0_real,1_fake}`, grouped by the top-level generator | `cnndetection` | +| `ufd` | fakes per generator, reals paired the UnivFD way (guided->imagenet, others->laion) | `universialfakedetection` | +| `pergen` | one `/{0_real,1_fake}` folder each, optional shared `--reals` | AIGIBench, FreqNet, Co-Spy-Bench | +| `pool` | one group: `/{real,fake}` | GenImage (per subset) | + +Useful flags: `--ckpt FILE` to score a different checkpoint (official vs SynthScope-trained side by side), `--select a,b` to +limit the generators, `--limit N` to cap images per class (quick checks), `--native-res` to score at the original resolution, +`--bs` for the batch size. Video: `python benchmark/bench_eval_video.py --model d3 --benchmark genvideo --data ... --tag ...`. + +Each run prints per-generator accuracy, AP and AUROC with the mean, writes `repro/bench_results/.json` and appends a +line to `repro/MASTER_RESULTS.jsonl`, so every reported number traces to a run. `python benchmark/aggregate_results.py` +rebuilds the combined table `repro/RESULTS_TABLE.md`. + +**Match the paper's protocol before judging a gap.** Read which subsets the paper evaluates and how it averages them. +For example CuRe reports the 20 full-image-synthesis subsets of AIGIBench, not all 25; scoring all 25 looked like a 7-point +miss until we used the paper's subsets. The input handling also matters: each detector's registered preprocessing mirrors its +paper's evaluation (a different resize or crop can destroy artifact-based detectors). + +### How a result reaches the Leaderboard tab + +1. The paper's numbers and the benchmark list for a detector live in [`repro/benchmark_specs/.json`](../repro/benchmark_specs/). + Add a spec file for a new detector (copy `cure.json`). +2. Add your measured numbers to the `VAL` table in [`interface/backend/build_repro_full.py`](../interface/backend/build_repro_full.py). + Every value must trace to a `repro/bench_results/*.json` file. +3. Rebuild and reload the page: + +```bash +python interface/backend/build_repro_full.py # writes interface/backend/repro_full.json +``` + +A row's status is computed, not typed: **Reproduced** (our own trained checkpoint is within the tolerance of the paper), +**Official weights match** (the authors' weights run through SynthScope match), **Below paper**, **Reference only** (the paper +reports no comparable number), **Not evaluated**. The tolerance is 2.0 points for accuracy and mAP, 2.5 for AUROC. + +--- + +## 4. Benchmark against an image generator + +The *Generator benchmarks* tab compares detectors on images from one generator. The recipe is the same for every generator: + +1. Take real photos (people are skipped) and write a prompt for each: a BLIP caption plus a photographic style suffix. +2. Let the generator **create a new image from the prompt alone**, so every pixel is synthesized (no editing of the photo). +3. Score the real and the generated images with every detector and rank by AUROC. + +```bash +# the same 100 real images and prompts for every generator, so generators are compared on identical content +python benchmark/generator_bench.py --generator flux2_klein --label "FLUX.2 [klein] 4B" \ + --reuse benchmark/generator_runs/reference/manifest.json --real-root /data/GenImage_full --real-root /data/CNNDetection \ + --out benchmark/generator_runs/flux2_klein_4b +``` + +The reference set ([`benchmark/generator_runs/reference/manifest.json`](../benchmark/generator_runs/reference)) lists 100 prompts and the +file names of the real images they were written for. The images themselves are third-party data and are not in the repo: +`--real-root` finds them by file name in your own GenImage and CNNDetection downloads (section 2), so your run uses the same +content as ours. Any real images you cannot match are skipped. + +- `--generator` is any key from [`interface/backend/generators.py`](../interface/backend/generators.py) that can create an + image from text (`flux2_klein`, `flux2_klein_9b`, `qwen_image_2_1`, `nanobanana`, `gpt_image`, `grok`, `flux`, `ideogram`). +- **Local open-weight generators** (`flux2_klein`, `flux2_klein_9b`, `qwen_image_2_1`) need a 24 GB GPU and a newer + `diffusers` than the main environment; the setup (`SYNTHSCOPE_GEN_PYTHONPATH`, gated-model licenses, `HF_TOKEN`) is in + [`interface/README.md`](../interface/README.md). They run as one batch with a single model load. +- **Hosted generators** need their API key in `.env` (`GEMINI_API_KEY`, `OPENAI_API_KEY`, `XAI_API_KEY`, `FAL_KEY`) and cost + money per image; the benchmark prints and skips any image the provider refuses. +- **Your own real images** instead of the reference set: `--real-glob "/data/real/*.jpg" --n 100` (repeat the flag for + several folders). Prompts are then captioned fresh, so numbers are only comparable between runs on the same images. +- `--detectors clip,safe,...` limits the detectors; `--skip-generate` re-scores images that are already there; an interrupted + run resumes because existing images are skipped. + +The run writes `manifest.json`, `results.json` and `RESULTS.md` (commit these) plus `real/` and `fake/` image folders +(git-ignored). The Generator benchmarks tab picks up every folder under `benchmark/generator_runs/` automatically; reload +the page after a run. + +**What the numbers mean.** AUROC is threshold-free (50 is chance, 100 perfect). *Fakes caught* and *Reals flagged* use the +detectors' 0.5 threshold, which is not tuned per generator, so accuracy can look poor even when AUROC is high. Real photos +are JPEG-origin and of varying size, while the generated images are 1024x1024 PNGs, which can help size- or +format-sensitive detectors; treat small differences with care. + +--- + +## 5. Time and hardware + +Measured on one RTX A5000 (24 GB) with 100 real + 100 generated images at 1024 px: + +| Step | Time | +| --- | --- | +| FLUX.2 [klein] 4B, generate 100 images | about 15 minutes | +| FLUX.2 [klein] 9B, generate 100 images | about 40 minutes | +| Qwen-Image-2.1, generate 100 images | about 70 minutes | +| Score all image detectors on the 200 images | 5 to 10 minutes | +| Download a dataset | minutes for CommGen15 (4 GB took about 2 minutes), hours for the largest (AIGIBench is about 220 GB) | +| A paper benchmark (`bench_eval.py`) | minutes for small sets, hours for GenImage-size sets (use `--limit` to try it) | + +Generation times are for a GPU that is not shared with other jobs on the same host; they grow when the node is busy. + +Host RAM matters for the local generators: give a job 48-64 GB (the larger models are held in host RAM and streamed to +the GPU). Models are downloaded once into the Hugging Face cache (tens of GB each), so point +`SYNTHSCOPE_GEN_HF_HOME` at a disk with room. + +--- + +## 6. Add your own + +- **A detector**: follow [`docs/ADD_DETECTOR.md`](ADD_DETECTOR.md); then add it to the Leaderboard as in section 3. +- **A dataset**: add `conf/dataset/image/.yaml` with the source (`hf_repo_id`, `source_url`, ...), make it follow the + layout in section 2, and add a profile to `bench_eval.py` if its grouping is new. +- **A generator**: add a function `call(image_png | None, prompt, size=None) -> png bytes` and one entry in `GENERATORS` in + `interface/backend/generators.py`. `image_png=None` means "create from the prompt alone". Then run section 4 with + `--generator `. +- **A number for the standardized leaderboard**: see [`benchmark/README.md`](../benchmark/README.md). diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md new file mode 100644 index 0000000000000000000000000000000000000000..25b052c0869f9b4222bd342f2f7a4896b64cc3ca --- /dev/null +++ b/docs/CAPABILITIES.md @@ -0,0 +1,171 @@ +# SynthScope capabilities (orientation for an agent building a new detector) + +SynthScope is a **reproduction + benchmarking library for AI-generated *image and video* detection**. It +hosts many published detectors behind one interface, reproduces each on its own benchmark, and +serves a web UI + a Python/CLI API. Read this before adding a method, then follow +[`docs/ADD_DETECTOR.md`](ADD_DETECTOR.md) for the exact checklist. + +> Environment: everything runs in the **`SynthScope` conda env** — +> `/home/ssalehi/miniconda3/bin/conda run -n SynthScope `. Config is **Hydra** (`conf/`). + +--- + +## 1. What's already integrated (17 detectors) + +`synthscope.list_models()` / `from synthscope.api import list_specs`. Each is an `SynthScopeModel` with a +matching `conf/model//.yaml`, a `trainer.py._build_model` branch, an +`interface/backend/registry.py` `ModelSpec`, a per-model `README.md`, and a `tests/test__smoke.py`. + +| key | modality | predict_kind | weights | interp map | method | +|---|---|---|---|---|---| +| `clip` | image | ce | synthscope | – | UnivFD / CLIP linear probe | +| `cnn` | image | ce | synthscope | – | CNNDetection (ResNet-50) | +| `dinov2`,`dinov3`,`adapted_dino` | image | ce | synthscope | – | DINO probes / adapter-tuned | +| `npr` | image | bce_npr | official | npr_residual | Neighboring-Pixel Relationships | +| `safe` | image | ce | synthscope | safe_hf | High-freq DWT artifacts | +| `aide` | image | ce | official | aide_dct | ConvNeXt-XXL + DCT/SRM (native-res 5-view) | +| `fat_former` | image | ce | official | – | Forgery-aware adaptive transformer (CLIP) | +| `spai` | image | bce | synthscope* | spai_patch | Spectral, any-resolution (MFM ViT-B/16) | +| `co_spy` | image | bce | synthscope* | – | Semantic + artifact fusion (SigLIP + SD-VAE) | +| `pgc` | image | bce | official | pgc_peak | LoRA-DINOv2 + YCbCr residual, peak-guided | +| `cure` | image | ce | official | – | Frozen PE-Core L/14 + Q/V LoRA + source-response subspace (mixing-ratio regression) | +| `vib_net` | image | vib | synthscope | – | Variational information bottleneck (FLAVA) | +| `d3` | video | video | training-free | – | Disentangled detail dynamics (X-CLIP) | +| `restrav` | video | video | synthscope | – | Representation-trajectory geometry (DINOv2) | +| `stall` | video | video | training-free | – | Spatio-temporal whitened log-likelihoods (DINOv3) | + +`*` served via official weights remapped into the SynthScope class. **`predict_kind`** sets the score +convention the engine applies to `forward()` logits: `ce`→`softmax[:,1]`, `bce`→`sigmoid`, +`bce_npr` (NPR residual transform then sigmoid), `vib` (tuple out), `video` (clip in). + +--- + +## 2. The detector contract (what a new method must implement) + +`models///.py`: +```python +class SynthScopeModel(nn.Module): + def __init__(self, config: DictConfig): ... # build arch from config.ModelConfig + def forward(self, images) -> logits: ... # RAW logits, not probs; model-specific + # preprocessing (residual/DWT/DCT) goes INSIDE forward + def compute_loss(self, batch) -> {"loss", "outputs"} # outputs = [N,2] real/fake logits, used by training +``` +Repo rule: **all imports + helper classes at module top level** (no nested defs/imports). Vendor the +official arch pieces verbatim where possible and document deviations in the per-model README. + +To wire it in: `conf/model//.yaml` (`model_type: `), a branch in +`trainer.py._build_model` (dispatches on `model_type`), and a `ModelSpec` in the interface registry +(see `ADD_DETECTOR.md`). The web engine (`interface/backend/engine.py`) lazy-imports by `model_type`. + +--- + +## 3. Training infrastructure + +- **`trainer.py`** (`SynthScopeModelTrainer`) + **`main.py`** entrypoint; Hydra config in `conf/config.yaml` + with groups `model/`, `dataset/`, `trainer/`. Multi-GPU via 🤗 `accelerate` (`accelerate_config.yaml`). +- **`dataset.py`** — image/video dataset loaders; `_decode_rgb` feeds uint8 `[3,H,W]` tensors; + `sample_video_frames(..., edge_crop=)` for video. +- **`transformations.py`** — composable named transforms / augmentations: `resize_224`, + `resize_short_{256,384,448}`, `center_crop_{224,256,384,448}`, `random_crop_*`, `pad_random_crop_224` + (PGC), `gaussian_blur`, `jpeg_compression`, `random_horizontal/vertical_flip`, `random_rotation_180`, + `color_jitter_safe`, `to_tensor`/`to_dtype`. **Eval-transform fidelity is critical** — artifact/ + low-level detectors (SAFE/AIDE/SPAI/PGC/NPR) are wrecked by a wrong resize; match the official eval + exactly (see `repro/investigate/pgc_gap_investigation.md` and the SAFE crop-not-resize lesson). +- **`metrics.py`** — torchmetrics binary `Accuracy` (real/fake), `AveragePrecision` (AP), + `AUROC`. Threshold-free AP/AUROC are the headline metrics (esp. video, where per-gen real:fake + ratios make mAP degenerate — prefer AUROC there). + +--- + +## 4. Evaluation / benchmarking + +- **Image**: `benchmark/bench_eval.py --model --benchmark {forensynths|ufd|pergen|pool} --data [--ckpt ] [--native-res] --tag `. Auto-discovers `/{0_real,1_fake}` (or `{ai,nature}`, shared-reals) layouts; prints per-generator acc/AP/AUROC + mean; writes `repro/bench_results/.json`. +- **Video**: `benchmark/bench_eval_video.py --model {d3|restrav|stall} --benchmark {genvideo|fakeonly|pergen}`. +- `--ckpt` overrides the registry checkpoint (for official-vs-SynthScope side-by-side). +- `benchmark/aggregate_results.py`, `leaderboard.py`, `evaluate.py`, `schema.json` aggregate runs. +- Results live in `repro/`: `RESULTS_SUMMARY.md` (read first), `AGGREGATED_BENCHMARK.md`, + `BENCHMARK_COVERAGE_MATRIX.md`, `REPRODUCED_MODELS_BENCHMARKS.md`, `repro/bench_results/*.json`, + and per-method deep-dives in `repro/investigate/`. + +--- + +## 5. Datasets (explorer registry: `interface/backend/datasets.py`, `DATASET_DEFS`) + +Known image sets: `cnndetection` (ForenSynths/ProGAN), `genimage`, `genimage_full` (8-gen), +`universialfakedetection` (Ojha UFD), `aigibench`, `synthbuster`, `ditfake`, `diffusionforensics`, +`diffusion1kstep`, `aigcdetect`, `commgen15`, `cospy`/`cospy_subset`, `spai_ldm`, `freqnetdetection`. +Video: `genvideo_val`, `restrav_data`, `evalcrafter`, `genvidbench`, `videofeedback`, `comgenvid`. + +Each `DatasetDef` lists candidate roots; **`_resolve_root` is node-portable** — it remaps the configured +`/hddstore` path onto whatever scratch mount exists (`/hddstore`↔`/ssdstore`↔`/nvmestore`), overridable +with `SYNTHSCOPE_DATA_PREFIXES`. Labels are inferred from path tokens (`0_real`/`nature`→real, +`1_fake`/`ai`→fake). **Not every dataset is physically present on every node** — data is node-local +scratch; check what's staged (e.g. CNNDetection, GenImage, GenImage_full are on cn029 `/ssdstore`). + +--- + +## 6. The `synthscope` package (Python + CLI) + +```python +import synthscope +det = synthscope.load("pgc") # or load(key, checkpoint=PATH, device=...) +det.predict("img.png") # -> {fake_probability, verdict, confidence, interpret_map, ...} +synthscope.list_models(); synthscope.available_models() # registered / weights-present +``` +CLI: `synthscope list`, `synthscope detect --model `, `synthscope pull ` (fetch weights from the HF model +zoo), `synthscope push` (maintainer). Weights resolve under **`$SYNTHSCOPE_CKPT_DIR`** (default +`/hddstore/ssalehi/SynthScope/checkpoints`; override per node) or are auto-downloaded. + +--- + +## 7. Web interface (`interface/`, FastAPI + uvicorn) + +`bash interface/run_backend.sh` (`SYNTHSCOPE_PORT`, `SYNTHSCOPE_HOST`). Serves `frontend/dist` if built, else the +no-build `static/index.html`. Endpoints (`interface/backend/app.py`): `/api/models`, `/api/predict` +(image/video, lazy model load + LRU), `/api/datasets[.../sample]`, `/api/predict_dataset`, +`/api/score_batch`, `/api/leaderboard`, `/api/generator_benchmarks`, and a **SOTA-vs-detector** demo +(`/api/sota/{generators,real_image,compare}`) that generates an image from a prompt (`generators.py`, +API-keyed SOTA generators incl. local Ideogram) and scores real vs generated across detectors. +On-cluster launch scripts: `repro/webui_cn029_serve.sbatch` (GPU on free cees node, checkpoints staged +to `/ssdstore`), `repro/webui_cn023_cpu.sbatch`. Reach it via SSH tunnel through the `ivi-h1` login host. + +### Interpretability maps +The engine returns a per-detector artifact map for the 5 native-signal detectors (`spec.interpret`): +`safe_hf` (DWT high-freq band), `npr_residual` (pixel residual), `pgc_peak` (peak-guided residual +score), `aide_dct` (native-res DCT patch energy), `spai_patch` (per-patch spectral fake-evidence). Each +yields **`interpret_map`** (bare jet heatmap) **+ `interpret_overlay`** (heatmap blended on the input, +transform-aligned). A new method exposes one by setting `interpret=""` on its `ModelSpec` and +adding a branch in `engine._interpret_map` that produces a 2-D saliency tensor. + +--- + +## 8. Trending pipeline & repo hygiene + +`repro/fetch_arxiv_trending.py` → `repro/generate_trending_readme.py` keep a curated +`interface/backend/trending.json` (image+video) and `TRENDING.md` (full list) + a short teaser in the README `` block + a web tab. +Daily rule: new detectors are integrated **on a `trending/` branch and pushed for review — never +auto-merged to `main`**; only `trending.json`/`TRENDING.md`/README docs go to `main` from the pipeline. Never commit +weights (`*.pth`), `.env`, or caches. + +--- + +## 9. Cluster cheat-sheet (IvI SLURM) + +- **cn023** (`quva`/`quvausers`, A6000) holds all detector checkpoints + most datasets on node-local + **`/hddstore`**; GPUs are often booked for days. `SYNTHSCOPE_CKPT_DIR=/hddstore/ssalehi/SynthScope/checkpoints`. +- **cees** (`ceesusers`: cn029 etc., a5000, free) and **cees6000** (cn030/31, rtx6000) use node-local + `/ssdstore` / `/nvmestore`. To run there, stage checkpoints/data from cn023 (`rsync -aL`, dereference + the symlinks) — note `pam_slurm_adopt` only allows ssh to a node where you hold an active job. +- `/ivi/zfs/...` is the shared home (24 GiB user quota — don't park big checkpoints there); + HF cache is shared at `/ivi/.../SynthScope/.cache_shared/hf` (`HF_HOME`). DINOv3 weights gated, under + `dinov3_checkpoints/` (`DINO_V3_WEIGHTS`); SPAI MFM backbone via `SYNTHSCOPE_MFM_CKPT`. + +--- + +## 10. Starting a new method — TL;DR + +1. Confirm it passes both trending rules (public repo + strong results). Read `ADD_DETECTOR.md`. +2. `git checkout -b trending/`; add the 5 files (model, conf, trainer dispatch, registry ModelSpec, README) + smoke test. +3. **Match the official eval transform exactly** in the `ModelSpec` (resize vs crop, norm) — this is the #1 silent-failure source. +4. Reproduce on the paper's benchmark with official weights via `bench_eval.py --ckpt`; fix `remap_` for key mismatches; record per-gen AP/AUROC in the Paper/SynthScope_official/SynthScope_train schema. +5. (Optional) add an interpretability map. Push the branch; do **not** merge to `main`. diff --git a/docs/images/datasets.jpg b/docs/images/datasets.jpg new file mode 100644 index 0000000000000000000000000000000000000000..663d52a90eee6dc389da8c3448a713257bfe955f --- /dev/null +++ b/docs/images/datasets.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53efa6b1bc5b695d739fea802475fe5357053206a7701100326b7879973e897 +size 293108 diff --git a/docs/images/detect.jpg b/docs/images/detect.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b048fd7de57833b7724b777be1377da84409f224 --- /dev/null +++ b/docs/images/detect.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdcb9d6574073127b4705007cfa4f042206dfbd35d90090315e93dbb21d88255 +size 176202 diff --git a/docs/images/genbench.jpg b/docs/images/genbench.jpg new file mode 100644 index 0000000000000000000000000000000000000000..af779463e970f534ac772dfe72cc54905eb5c0f1 --- /dev/null +++ b/docs/images/genbench.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c01c40058fd46bdec73d59bfbaaa0c41418e4c23ba3a05098c84b0f8ac504f0f +size 159387 diff --git a/docs/images/generator.jpg b/docs/images/generator.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4688fac097b8cc56bb3e1a118804f156e548a1f6 --- /dev/null +++ b/docs/images/generator.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eed545e7f722536d4703ad20bb41e57787870192b15d93d88d92e4d5744bfb25 +size 236466 diff --git a/docs/images/leaderboard.jpg b/docs/images/leaderboard.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a74c7b60f4b6d321c5b23e2e9be6aef09a0b8c12 --- /dev/null +++ b/docs/images/leaderboard.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c30708b9988c3f8c1d94763808a90e567ffa2b99d5e527d736d7105250c239f6 +size 161083 diff --git a/docs/sota_detection_papers.md b/docs/sota_detection_papers.md new file mode 100644 index 0000000000000000000000000000000000000000..9bd424b88737872f3515657c3cb8e1ab4dd57f9a --- /dev/null +++ b/docs/sota_detection_papers.md @@ -0,0 +1,50 @@ +# SOTA AI-Generated Media Detection — papers worth looking into + +Curated for the SynthScope project (AI-generated **image** and **video** detection). ✓ = already +implemented in this repo. Venues are best-effort; verify before citing. + +## Already in SynthScope +- **Image:** CNNDetection ✓, UnivFD/CLIP ✓, NPR ✓, FatFormer ✓, AIDE ✓, CO-SPY ✓, SAFE ✓, + SPAI ✓, VIB-Net ✓ (+ DINOv2 / CLIP / ResNet linear-probe baselines). +- **Video:** D3 ✓, ReStraV ✓. + +## AI-generated IMAGE detection +| Method | Paper | Venue | Idea | +|---|---|---|---| +| CNNDetection ✓ | CNN-generated images are surprisingly easy to spot… for now (Wang et al.) | CVPR 2020 | ResNet-50 on ProGAN + blur/JPEG aug; strong cross-GAN transfer. | +| UnivFD ✓ | Towards Universal Fake Image Detectors that Generalize Across Generative Models (Ojha et al.) | CVPR 2023 | Frozen CLIP features + linear/NN probe. | +| DIRE | DIRE for Diffusion-Generated Image Detection (Wang et al.) | ICCV 2023 | Reconstruction error under a diffusion model. | +| LGrad | Learning on Gradients (Tan et al.) | CVPR 2023 | Gradient-based generalized artifact representation. | +| NPR ✓ | Rethinking the Up-Sampling Operations… (Tan et al.) | CVPR 2024 | Neighboring-pixel residuals from up-sampling. | +| FatFormer ✓ | Forgery-Aware Adaptive Transformer (Liu et al.) | CVPR 2024 | Frozen CLIP + forgery-aware adapters + language-guided alignment. | +| FreqNet | Frequency-Aware Deepfake Detection (Tan et al.) | AAAI 2024 | High-frequency representation learning. | +| DRCT | Diffusion Reconstruction Contrastive Training (Chen et al.) | ICML 2024 | Reconstruct-then-contrast for diffusion images. | +| RINE | Leveraging Representations from Intermediate Encoder Blocks (Koutlis & Papadopoulos) | ECCV 2024 | Uses intermediate CLIP features, not just CLS. | +| Cozzolino et al. | Raising the Bar of AI-generated Image Detection with CLIP | CVPRW 2024 | Strong, simple CLIP-feature baseline + protocol. | +| FID / "Breaking Semantic Artifacts" | Breaking Semantic Artifacts for Generalized AIGI Detection (Zheng et al.) | NeurIPS 2024 | Local-feature method; the strongest image baseline in the D3 paper. | +| AIDE ✓ | AIDE: AI-generated Image Detection with Hybrid Features (Yan et al.) | ICLR 2025 | ConvNeXt semantics + DCT/SRM low-level patches. | +| CO-SPY ✓ | Combining Semantic and Pixel Features (Cheng et al.) | CVPR 2025 | SigLIP semantics + VAE-residual artifacts, adaptive fusion. | +| SAFE ✓ | Improving SID Towards Generalization: An Image Transformation Perspective (Li et al.) | KDD 2025 | Lightweight DWT high-freq detector + augmentation recipe. | +| VIB-Net ✓ | Universal AIGI Detection by Variational Information Bottleneck (Zhang et al.) | CVPR 2025 | VIB on CLIP features to suppress shortcut cues. | + +## AI-generated VIDEO detection +| Method | Paper | Venue | Idea | +|---|---|---|---| +| DeMamba | AI-Generated Video Detection on Million-Scale GenVideo (Chen et al.) | 2024 (arXiv 2405.19707) | GenVideo benchmark + Mamba temporal module; strong trained baseline. | +| DeCoF | Generated Video Detection via Frame Consistency (Ma et al.) | 2024 (arXiv 2402.02085) | Temporal frame-consistency features. | +| D3 ✓ | Training-Free AIGV Detection Using Second-Order Features (Zheng et al.) | ICCV 2025 | Training-free; std of 2nd-order inter-frame feature differences. | +| ReStraV ✓ | AI-Generated Video Detection via Perceptual Straightening (Internò et al.) | 2025 (arXiv 2507.00583) | DINOv2 trajectory curvature/distance + small classifier. | +| AIGVDet | Spatial-Temporal Anomaly Learning (Bai et al.) | PRCV 2024 | Optical-flow + spatial anomaly streams. | +| DuB3D | Distinguish Any Fake Videos (Ji et al.) | 2024 (arXiv 2405.15343) | Large-scale data + motion features. | +| "Turns out I'm not real" | Towards Robust Detection of AI-Generated Videos (Liu et al.) | 2024 (arXiv 2406.09601) | DIRE residuals + CNN/LSTM, robustness focus. | + +Face-deepfake-video methods often cited as baselines (less relevant to *generative* video, but useful): FTCN (ICCV 2021), TALL (ICCV 2023), MINTIME (TIFS 2024), STIL (ACM MM 2021). + +## Benchmarks / datasets +- **Image:** GenImage, ForenSynths (CNNDetection), UniversalFakeDetect, DiffusionForensics, Co-Spy-Bench, AIGIBench, DiTFake. +- **Video:** GenVideo (DeMamba), GenVidBench, VidProM, EvalCrafter, VideoPhy / VideoPhy-2, Physics-IQ. + +## Suggested starting points +1. **Cross-modal trend:** frozen foundation features (CLIP/DINOv2/SigLIP) + light heads now dominate both image and video — UnivFD → FatFormer → AIDE/CO-SPY (image), ReStraV/D3 (video). +2. **Training-free direction:** D3 (video) and the local-feature image methods suggest detection signals that need no generated training data — worth exploring for generalization. +3. **For video specifically:** DeMamba (benchmark + trained SOTA) vs. D3/ReStraV (cheap, training-free/light) is the key comparison; GenVideo is the common benchmark. diff --git a/experiments/image/adapted_dinov2_vitl14.sh b/experiments/image/adapted_dinov2_vitl14.sh new file mode 100644 index 0000000000000000000000000000000000000000..1939aad6a7b304f97160b7c2c5e229cb46c88bb9 --- /dev/null +++ b/experiments/image/adapted_dinov2_vitl14.sh @@ -0,0 +1,14 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2,3,4 +export OMP_NUM_THREADS=1 + +accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py \ + ModelConfig.bottleneck_dim=64 \ + ModelConfig.model_type=dino \ + ModelConfig.use_adapter=true \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_dinov2] \ + ModelConfig.backbone_name=dinov2_vitl14 \ No newline at end of file diff --git a/experiments/image/aide.sh b/experiments/image/aide.sh new file mode 100644 index 0000000000000000000000000000000000000000..30fd7ceb728abff28148f1c90dc235da22511a28 --- /dev/null +++ b/experiments/image/aide.sh @@ -0,0 +1,19 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/aide \ + model@ModelConfig=image/aide \ + dataset@DatasetConfig=image/genimage \ + DatasetConfig.train_batch_size=16 \ + DatasetConfig.val_batch_size=16 \ + DatasetConfig.test_batch_size=16 \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_256,to_dtype] \ + logger.name=aide \ + logger.mode=online \ + logger.tags=[SynthScope,AIDE,genimage] diff --git a/experiments/image/clip_vitl14.sh b/experiments/image/clip_vitl14.sh new file mode 100644 index 0000000000000000000000000000000000000000..353aedd90d33758c8037d79b5c2e8776d78103f1 --- /dev/null +++ b/experiments/image/clip_vitl14.sh @@ -0,0 +1,17 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2,3,4 +export OMP_NUM_THREADS=1 + +# accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py +# accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py + +accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py \ + ModelConfig.bottleneck_dim=64 \ + ModelConfig.model_type=clip \ + ModelConfig.use_adapter=false \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_clip] \ + ModelConfig.backbone_name=CLIP:ViT-L/14 \ No newline at end of file diff --git a/experiments/image/co_spy.sh b/experiments/image/co_spy.sh new file mode 100644 index 0000000000000000000000000000000000000000..9b04c60169c5478fdf7f41a11661737e1a246925 --- /dev/null +++ b/experiments/image/co_spy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/co_spy \ + model@ModelConfig=image/co_spy \ + DatasetConfig.train_batch_size=16 \ + DatasetConfig.val_batch_size=16 \ + DatasetConfig.test_batch_size=16 \ + DatasetConfig.train_transform_list=[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype] \ + DatasetConfig.eval_transform_list=[resize_short_384,center_crop_384,to_dtype] \ + logger.name=co_spy \ + logger.mode=online \ + logger.tags=[SynthScope,CoSpy,genimage] diff --git a/experiments/image/dinov2_vitl14.sh b/experiments/image/dinov2_vitl14.sh new file mode 100644 index 0000000000000000000000000000000000000000..6092b5c2760f8280eeeaecf88997948f9d1a0bb2 --- /dev/null +++ b/experiments/image/dinov2_vitl14.sh @@ -0,0 +1,17 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2,3,4 +export OMP_NUM_THREADS=1 + +# accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py +# accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py + +accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py \ + ModelConfig.bottleneck_dim=64 \ + ModelConfig.model_type=dinov2 \ + ModelConfig.use_adapter=false \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_dinov2] \ + ModelConfig.backbone_name=dinov2_vitl14 \ No newline at end of file diff --git a/experiments/image/dinov3_vitl14.sh b/experiments/image/dinov3_vitl14.sh new file mode 100644 index 0000000000000000000000000000000000000000..7ead0385236aed6b142b951020e7c867e89cd84b --- /dev/null +++ b/experiments/image/dinov3_vitl14.sh @@ -0,0 +1,17 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +# accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py +# accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py + +accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + ModelConfig.bottleneck_dim=64 \ + ModelConfig.model_type=dinov3 \ + ModelConfig.use_adapter=false \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype,normalize_dinov3] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_256,to_dtype,normalize_dinov3] \ + ModelConfig.backbone_name=dinov3_vitl16 \ No newline at end of file diff --git a/experiments/image/fat_former.sh b/experiments/image/fat_former.sh new file mode 100644 index 0000000000000000000000000000000000000000..346b0e48f866ef2e6945d44184a1259bf5ac0bd3 --- /dev/null +++ b/experiments/image/fat_former.sh @@ -0,0 +1,23 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +# Official FatFormer (Michel-liu/FatFormer, utils/dataset.py) normalizes with **ImageNet** +# mean/std -- NOT CLIP stats -- despite the CLIP ViT-L/14 backbone. `normalize_dinov2` is the +# ImageNet Normalize in this repo, so we use it here to match the official preprocessing (and the +# deployed official-weights eval, which reproduces 98.39 acc / 99.70 mAP with ImageNet norm). +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/fat_former \ + model@ModelConfig=image/fat_former \ + dataset@DatasetConfig=image/aigibench \ + DatasetConfig.train_batch_size=32 \ + DatasetConfig.val_batch_size=32 \ + DatasetConfig.test_batch_size=32 \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_dinov2] \ + logger.name=fat_former \ + logger.mode=online \ + logger.tags=[SynthScope,FatFormer,aigibench] diff --git a/experiments/image/npr.sh b/experiments/image/npr.sh new file mode 100644 index 0000000000000000000000000000000000000000..1fc9a15148981bc0581f146eb449814dfd36e2ad --- /dev/null +++ b/experiments/image/npr.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +# NPR trains a small ResNet from scratch on neighboring-pixel residuals; feed [0,1] images (to_dtype only). +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/npr \ + model@ModelConfig=image/npr \ + dataset@DatasetConfig=image/genimage \ + DatasetConfig.train_batch_size=128 \ + DatasetConfig.val_batch_size=512 \ + DatasetConfig.test_batch_size=512 \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype] \ + logger.name=npr \ + logger.mode=online \ + logger.tags=[SynthScope,NPR,genimage] diff --git a/experiments/image/pgc.sh b/experiments/image/pgc.sh new file mode 100644 index 0000000000000000000000000000000000000000..3f2e23d000d9c2e4a29372e3ac750d11602daecf --- /dev/null +++ b/experiments/image/pgc.sh @@ -0,0 +1,27 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +# PGC (Zhou et al., 2026): LoRA-tuned DINOv2-Large RGB stream + YCbCr-quantization-residual CNN +# stream, fused by peak-guided calibration. The residual is computed INSIDE the model, so the +# dataset feeds a normal ImageNet-normalized image (normalize_dinov2 = what DINOv2 expects). The +# residual is resolution-sensitive, so we use CROP-only transforms (no resize) to preserve the +# per-pixel quantization pattern. Trained here on ProGAN (CNNDetection); swap the dataset for +# GenImage to match the paper's SDv1.4 setting. +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/pgc \ + model@ModelConfig=image/pgc \ + dataset@DatasetConfig=image/cnndetection \ + ModelConfig.backbone_name=facebook/dinov2-large \ + DatasetConfig.train_batch_size=32 \ + DatasetConfig.val_batch_size=64 \ + DatasetConfig.test_batch_size=64 \ + DatasetConfig.train_transform_list=[random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ + DatasetConfig.eval_transform_list=[center_crop_224,to_dtype,normalize_dinov2] \ + metrics=[real_accuracy,fake_accuracy,AP,AUC] \ + logger.name=pgc \ + logger.mode=online \ + logger.tags=[SynthScope,PGC,cnndetection] diff --git a/experiments/image/safe.sh b/experiments/image/safe.sh new file mode 100644 index 0000000000000000000000000000000000000000..92cbab81c1520427c270931a42b298d8ba03c52a --- /dev/null +++ b/experiments/image/safe.sh @@ -0,0 +1,27 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/safe \ + dataset@DatasetConfig=image/aigibench \ + ModelConfig.model_type=safe \ + ModelConfig.freeze_backbone=false \ + DatasetConfig.train_batch_size=64 \ + DatasetConfig.val_batch_size=64 \ + DatasetConfig.test_batch_size=64 \ + DatasetConfig.train_transform_list=[random_crop_256_pad,random_horizontal_flip,random_rotation_180,color_jitter_safe,to_dtype,random_mask_safe] \ + DatasetConfig.eval_transform_list=[center_crop_256,to_dtype] \ + TrainerConfig.epochs=100 \ + TrainerConfig.lr=0.0005 \ + TrainerConfig.weight_decay=0.05 \ + TrainerConfig.label_smoothing=0.1 \ + TrainerConfig.warmup.mode=on \ + TrainerConfig.warmup.warmup_ratio=0.2 \ + TrainerConfig.warmup.lr_scheduler=cosine \ + logger.name=safe \ + logger.mode=online \ + logger.tags=[SynthScope,SAFE,aigibench] diff --git a/experiments/image/spai.sh b/experiments/image/spai.sh new file mode 100644 index 0000000000000000000000000000000000000000..258edad86f9fdf25acb455113aa209467b51a913 --- /dev/null +++ b/experiments/image/spai.sh @@ -0,0 +1,19 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +# SPAI expects [0,1] images (to_dtype only); ImageNet normalization is applied inside the model. +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/spai \ + model@ModelConfig=image/spai \ + DatasetConfig.train_batch_size=32 \ + DatasetConfig.val_batch_size=32 \ + DatasetConfig.test_batch_size=32 \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype] \ + logger.name=spai \ + logger.mode=online \ + logger.tags=[SynthScope,SPAI,genimage] diff --git a/experiments/image/vib_net.sh b/experiments/image/vib_net.sh new file mode 100644 index 0000000000000000000000000000000000000000..701b14276029b36f38cdfabae6ca7b2bb8b81f4e --- /dev/null +++ b/experiments/image/vib_net.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=1,2 +export OMP_NUM_THREADS=1 + +# VIB-Net uses a frozen CLIP ViT-L/14 backbone (CLIP normalization, 224 crops). +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + trainer@TrainerConfig=image/vib_net \ + model@ModelConfig=image/vib_net \ + dataset@DatasetConfig=image/genimage \ + DatasetConfig.train_batch_size=128 \ + DatasetConfig.val_batch_size=512 \ + DatasetConfig.test_batch_size=512 \ + DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip] \ + DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_clip] \ + logger.name=vib_net \ + logger.mode=online \ + logger.tags=[SynthScope,VIBNet,genimage] diff --git a/experiments/video/d3.sh b/experiments/video/d3.sh new file mode 100644 index 0000000000000000000000000000000000000000..cb3c8e8b269fdc0ca4f7290c23710513916e6f0a --- /dev/null +++ b/experiments/video/d3.sh @@ -0,0 +1,28 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=0 +export OMP_NUM_THREADS=1 + +# D3 is training-free: a frozen CLIP ViT-B/16 extracts per-frame features, and the per-video +# second-order std becomes the detection score. Frames are decoded to 224x224 in [0,1] and +# normalized by the dataset transform pipeline (normalize_clip here, matched to the CLIP +# backbone -- switch to normalize_dinov2 if you set a DINOv2/ResNet backbone). AP/AUROC are the +# meaningful metrics. The video benchmark must be present under DatasetConfig.root_dir; +# this is a template (data download is out of scope). +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 main.py \ + trainer@TrainerConfig=video/d3 \ + model@ModelConfig=video/d3 \ + dataset@DatasetConfig=video/genvideo \ + ModelConfig.backbone_name=clip \ + ModelConfig.first_order=l2 \ + DatasetConfig.test_batch_size=8 \ + DatasetConfig.num_frames=16 \ + DatasetConfig.fps=8 \ + DatasetConfig.train_transform_list=[normalize_clip] \ + DatasetConfig.eval_transform_list=[normalize_clip] \ + metrics=[real_accuracy,fake_accuracy,AP,AUC] \ + logger.name=d3 \ + logger.mode=online \ + logger.tags=[SynthScope,D3,genvideo] diff --git a/experiments/video/restrav.sh b/experiments/video/restrav.sh new file mode 100644 index 0000000000000000000000000000000000000000..779a3afa71e30919da611654f8d8e7840bc939db --- /dev/null +++ b/experiments/video/restrav.sh @@ -0,0 +1,33 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=0 +export OMP_NUM_THREADS=1 + +# ReStraV trains a small MLP on DINOv2 ViT-S/14 trajectory-geometry features (distance + +# curvature). The DINOv2 backbone is frozen; only the MLP trains. Frames are decoded to +# 224x224 over a CENTER 2 s window (24 frames, BICUBIC) and fed to DINOv2 as RAW [0,1] with +# *no* ImageNet normalization -- this matches the official ReStraV repo verbatim +# (dinov2_features.preprocess = frames/255 + BICUBIC resize, no Normalize) and the deployed +# serving spec (registry video_norm = identity). Do NOT add normalize_dinov2 here: the +# detector is served raw [0,1], so normalizing only at train time creates a train/serve +# mismatch that collapses cross-generator transfer. Official optimizer is plain Adam (no +# weight decay), 10 epochs -- matched below. The video benchmark must be present under +# DatasetConfig.root_dir; this is a template (data download is out of scope). +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 main.py \ + trainer@TrainerConfig=video/restrav \ + model@ModelConfig=video/restrav \ + dataset@DatasetConfig=video/vidprom \ + TrainerConfig.epochs=10 \ + TrainerConfig.weight_decay=0.0 \ + DatasetConfig.train_batch_size=8 \ + DatasetConfig.val_batch_size=8 \ + DatasetConfig.test_batch_size=8 \ + DatasetConfig.num_frames=24 \ + DatasetConfig.train_transform_list=[] \ + DatasetConfig.eval_transform_list=[] \ + metrics=[real_accuracy,fake_accuracy,AP,AUC] \ + logger.name=restrav \ + logger.mode=online \ + logger.tags=[SynthScope,ReStraV,vidprom] diff --git a/experiments/video/stall.sh b/experiments/video/stall.sh new file mode 100644 index 0000000000000000000000000000000000000000..cd5f0f90d6390e1974f20578dc3cd0cb175c6dd8 --- /dev/null +++ b/experiments/video/stall.sh @@ -0,0 +1,32 @@ +#!/bin/bash +export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton +export HF_HOME=/nvmestore/ssalehi/.cache/huggingface +export TORCH_HOME=/nvmestore/ssalehi/.cache/torch +export CUDA_VISIBLE_DEVICES=0 +export OMP_NUM_THREADS=1 + +# STALL (Ben Hayun et al., CVPR'26) is training-free: a frozen DINOv3 ViT-L/16 extracts +# per-frame features, which are scored against a VATEX real-video calibration via two whitened +# isotropic-Gaussian likelihood branches (spatial + temporal). Frames are decoded square at 224 +# in [0,1] and normalized with ImageNet stats -- normalize_dinov2 here, because the official +# DINOv3 transform uses ImageNet mean/std (NOT normalize_dinov3). AP/AUROC are the meaningful +# metrics. The video benchmark must be present under DatasetConfig.root_dir. +# +# DINOv3 ViT-L/16 backbone is GATED: point these at your local clone + weights. +export DINO_V3_REPO_DIR=${DINO_V3_REPO_DIR:-/home/ssalehi/.cache/torch/hub/facebookresearch_dinov3} +export DINO_V3_WEIGHTS=${DINO_V3_WEIGHTS:-/hddstore/ssalehi/SynthScope/dinov3_checkpoints/dinov3_vitl16_pretrain_lvd1689m-8aa4cbdd.pth} + +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 main.py \ + trainer@TrainerConfig=video/stall \ + model@ModelConfig=video/stall \ + dataset@DatasetConfig=video/genvideo \ + ModelConfig.backbone_name=dinov3_vitl16 \ + DatasetConfig.test_batch_size=8 \ + DatasetConfig.num_frames=16 \ + DatasetConfig.frame_size=224 \ + DatasetConfig.train_transform_list=[normalize_dinov2] \ + DatasetConfig.eval_transform_list=[normalize_dinov2] \ + metrics=[real_accuracy,fake_accuracy,AP,AUC] \ + logger.name=stall \ + logger.mode=online \ + logger.tags=[SynthScope,STALL,genvideo] diff --git a/interface/.gitignore b/interface/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..88302acee5ffc10f77f0cb94f5fb5637feab575e --- /dev/null +++ b/interface/.gitignore @@ -0,0 +1,14 @@ +# Frontend build artifacts +frontend/node_modules/ +frontend/dist/ +frontend/.vite/ + +# Python +__pycache__/ +*.pyc + +# Local sample uploads +samples/*.mp4 +samples/*.png +samples/*.jpg +samples/*.jpeg diff --git a/interface/README.md b/interface/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7dd9f435b4050e908a0190c3a90c8fa1d68534ea --- /dev/null +++ b/interface/README.md @@ -0,0 +1,196 @@ +# SynthScope Web Interface + +A web app to run the SynthScope detectors on an uploaded **image or video** and get the probability +that it is **AI-generated (fake)** vs **real**. Pick any detector from a menu; the backend loads +the corresponding model + checkpoint and runs real inference. + +``` +interface/ +├── backend/ FastAPI service (model registry, inference engine, media decoding) +│ ├── app.py HTTP API + static serving +│ ├── registry.py per-detector config / preprocessing / checkpoint / output convention +│ ├── engine.py lazy model build, checkpoint load, LRU cache, predict +│ ├── media.py image/video decoding (reuses the repo's own decoders) +│ └── config.py paths, caches, device, limits (env-overridable) +├── frontend/ Vite + React SPA (build where Node is available) +├── static/index.html no-build React page (CDN) the backend serves by default +├── scripts/ checkpoint collection / training / promotion +├── requirements.txt web deps (added on top of the `SynthScope` conda env) +└── run_backend.sh launch the backend +``` + +## Quick start (on the GPU node, e.g. cn023) + +```bash +# 1. one-time: web deps into the SynthScope conda env +conda run -n SynthScope pip install -r interface/requirements.txt + +# 2. assemble the central checkpoint dir (symlinks the available official weights) +conda run -n SynthScope python interface/scripts/collect_checkpoints.py + +# 3. run the backend (serves the no-build UI at http://HOST:8000) +bash interface/run_backend.sh +``` + +Then open `http://:8000` (use SSH port-forwarding from your laptop, e.g. +`ssh -L 8000:cn023:8000 …`). The browser fetches React from a CDN; the model inference happens +on the node. + +### Optional: the full Vite/React build +Where Node is available (the cluster nodes don't have it): +```bash +cd interface/frontend +npm install +npm run build # emits frontend/dist, which the backend serves automatically +# or for live dev with hot reload (proxies /api to the backend on :8000): +npm run dev +``` + +## How it works + +* Every detector is a SynthScope `nn.Module` built exactly like `main.py` does, via + `config_utils.compose_config()` + the same `model_type` dispatch as + `trainer._build_model`. +* The checkpoint is loaded into that module. Official weights (FatFormer, AIDE, NPR) are remapped + to the SynthScope state-dict layout (the verified remaps from `repro/*_eval.py`); trained-from-scratch + SynthScope checkpoints load directly. +* Preprocessing matches each model's training/eval recipe (see `registry.py`). +* The fake probability is the model's native positive-class score: `softmax(logits)[fake]` for the + cross-entropy heads, `sigmoid(logit)` for the BCE heads. Label index **1 = fake** + (`dataset.py:_get_label`). +* **Video**: native video detectors (D3, ReStraV, STALL) decode a clip and score it directly; image + detectors sample frames (default 16), score each as an image, and report the **mean** P(fake) + plus a per-frame breakdown. +* Models load lazily on first request and are kept in a small LRU (`SYNTHSCOPE_MAX_RESIDENT_MODELS`, + default 2) so the GPU is not exhausted by the large backbones. + +## Detectors (16) & helper scripts + +13 image (`clip`, `cnn`, `dinov2`, `dinov3`, `adapted_dino`, `safe`, `fat_former`, `co_spy`, `aide`, +`spai`, `vib_net`, `npr`, `pgc`) + 3 video (`d3`, `restrav`, `stall`). Per-model held-out numbers and the +checkpoint-zoo table are in the top-level [README](../README.md#web-interface--checkpoint-zoo). + +| script | purpose | +| --- | --- | +| `scripts/collect_checkpoints.py` | symlink official weights into `$SYNTHSCOPE_CKPT_DIR` + write `MANIFEST.md` | +| `scripts/train_model.sh [epochs]` | train a SynthScope-class detector (saving on) | +| `scripts/promote_checkpoint.py ` | publish the best trained epoch as `.pth` | +| `scripts/verify_model.py ` | quick held-out check on ForenSynths | +| `scripts/build_restrav_data.py` | stage GenVideo into the image-style `//{0_real,1_fake}` layout | +| `scripts/calibrate_d3.py` | calibrate D3's score→probability | +| `scripts/restrav_repro.py` | **faithful** ReStraV reproduction (in-distribution + cross-generator) | + +Notes: +* **STALL** is training-free; it ships the official VATEX calibration npz and can recompute it on your own + real videos (`ModelConfig.recompute_params=true`). It and the **`dinov3`** probe need the gated **DINOv3 + ViT-L/16** weights (set `DINO_V3_WEIGHTS` / `DINO_V3_WEIGHTS_DIR`). +* **ReStraV** uses raw `[0,1]` input + standardized 21-d features (matching the official repo); see the + README footnote on in-distribution vs cross-generator results. + +## API + +| Method | Path | Body | Returns | +| --- | --- | --- | --- | +| GET | `/api/health` | – | status, device, checkpoints dir | +| GET | `/api/models` | – | detectors + availability | +| POST | `/api/predict` | multipart `file`, `model` | `{fake_probability, real_probability, verdict, …}` | +| GET | `/api/datasets` | – | on-disk datasets + per-model training dataset | +| GET | `/api/datasets/{key}/sample` | query `split`, `n`, `seed` | random balanced batch `{items:[{rel,label,modality}]}` | +| GET | `/api/datasets/{key}/media` | query `rel`, `thumb` | the file (or a JPEG thumbnail / video middle-frame) | +| POST | `/api/predict_dataset` | form `dataset`, `rel`, `model` | prediction on a dataset file + `ground_truth`, `correct` | + +```bash +curl -s -X POST http://localhost:8000/api/predict \ + -F model=fat_former -F file=@/path/to/image.png | python -m json.tool +``` + +The UI has two tabs: **Detect** (upload an image/video) and **Data Explorer** (pick a dataset — +the one used to train the chosen detector is starred — sample a random batch of thumbnails, click any +item to run the detector and see prediction vs. ground truth, and "↻ New batch" to resample). Datasets +are auto-discovered from disk (`backend/datasets.py`), so only those present on the node appear. + +## Checkpoints + +Central directory (env `SYNTHSCOPE_CKPT_DIR`, default `/hddstore/ssalehi/SynthScope/checkpoints`). The menu +shows a model as available only when its checkpoint file is present (D3 is training-free and always +available). See `MANIFEST.md` there for status. + +* `collect_checkpoints.py` — symlink available official weights to canonical names + write manifest. +* `train_model.sh [epochs]` — train a missing model with saving enabled. +* `promote_checkpoint.py ` — publish the best trained epoch as `.pth`. + +## Configuration (env vars) + +| Var | Default | Meaning | +| --- | --- | --- | +| `SYNTHSCOPE_CKPT_DIR` | `/hddstore/ssalehi/SynthScope/checkpoints` | central checkpoint directory | +| `SYNTHSCOPE_CACHE_DIR` | `/hddstore/ssalehi/SynthScope/.cache` | HF/torch backbone cache | +| `SYNTHSCOPE_MAX_RESIDENT_MODELS` | `2` | models kept in GPU memory (LRU) | +| `SYNTHSCOPE_VIDEO_IMAGE_FRAMES` | `16` | frames sampled when an image detector runs on a video | +| `SYNTHSCOPE_MAX_UPLOAD_MB` | `200` | upload size limit | +| `SYNTHSCOPE_PORT` / `SYNTHSCOPE_HOST` | `8000` / `0.0.0.0` | backend bind | + +## Deployment notes + +* Checkpoints, datasets and backbone caches live on **node-local** storage (cn023 `/hddstore`), so + run the backend on that node, or set `SYNTHSCOPE_CKPT_DIR`/`SYNTHSCOPE_CACHE_DIR` to wherever the weights are. +* First use of a model downloads its pretrained backbone (CLIP/DINOv2/SigLIP/…) into the cache. +* GPU is auto-detected; falls back to CPU (slow for the large backbones). + +## Configuring SOTA generator API keys + +The "SOTA vs Detectors" demo calls external image generators (Nano Banana / GPT Image / Grok / FLUX). +Keys are read from environment variables — **never commit them**: + +```bash +cp .env.example .env # at the repo root; .env is gitignored +# edit .env and fill in the keys you have: +# GEMINI_API_KEY=... OPENAI_API_KEY=... XAI_API_KEY=... FAL_KEY=... +``` + +The backend auto-loads `.env` at startup (`interface/backend/config.py`). Real shell exports or a +secrets manager override the file. A generator whose key is unset shows greyed-out "(no API key)" in +the UI; the rest of the demo (real-image serving, detector scoring) works without any key. For +production, prefer real env vars / a secrets manager over the `.env` file. + +**Local Ideogram 4.0 (open-weight, no paid API key):** needs `Ideogram4Pipeline`, currently only in diffusers `main`: +```bash +pip install "git+https://github.com/huggingface/diffusers.git" +``` +and a free `HF_TOKEN` (once) to download the license-gated nf4 weights (accept at hf.co/ideogram-ai/ideogram-4-nf4-diffusers). It then runs locally on a 24 GB GPU. + +### Local open-weight generators: FLUX.2 [klein] 4B and Qwen-Image-2.1 (no paid API key) + +Two image-**conditioned** generators run locally and edit the uploaded photo into a new image: + +| key | model | size | license | notes | +|---|---|---|---|---| +| `flux2_klein` | `black-forest-labs/FLUX.2-klein-4B` | 24 GB | Apache-2.0 | distilled, 4 steps (~1 min incl. load) | +| `qwen_image_2_1` | `Qwen/Qwen-Image-2.1` (7B unified generation + editing) | 33 GB | **Qwen Research License (research use only)** | 40 steps (~1.5 min incl. load) | + +They need a newer `diffusers` than the shared env (`QwenImage21Pipeline`), so they run in an **isolated +subprocess** (`interface/backend/local_gen.py`) with their own library path — the shared `SynthScope` env is left +untouched. One-time setup (no env changes): + +```bash +G=/path/to/gen_env && mkdir -p $G +pip install --no-deps --target $G git+https://github.com/huggingface/diffusers +pip install --no-deps --target $G "huggingface_hub>=1.12,<2" hf_xet # diffusers main needs hub 1.x; hub 2.x is rejected by transformers +export SYNTHSCOPE_GEN_PYTHONPATH=$G # prepended to the child's PYTHONPATH +export SYNTHSCOPE_GEN_HF_HOME=/node-local/hf_cache # 57 GB of weights: keep them off the shared quota +``` + +Both are ungated, but a `HF_TOKEN` in `.env` is passed through. A generator shows as available only when +`SYNTHSCOPE_GEN_PYTHONPATH` points at an existing directory **and** a GPU is visible. + +How it fits a 24 GB GPU next to the detectors: the web engine first releases its resident detectors +(`Engine.release_gpu`, they reload lazily); then the child runs one image and exits, freeing everything. +FLUX.2 uses `enable_model_cpu_offload`. Qwen-Image-2.1 (33 GB, would need >28 GB of host RAM that way) is +loaded **in stages** straight to the GPU — text encoder (17 GiB) → dropped → 7B transformer (14 GiB) → moved to +host RAM just before a single-pass VAE decode — so host RAM stays small. (VAE *tiling* is deliberately not used: +tiling the encode of the conditioning image corrupts it, and tiled decode leaves visible seams a detector could +key on.) Generation runs off the event loop, and local runs are serialized by a lock. + +Other FLUX variants (`FLUX.2-klein-9B`, `FLUX.1-Kontext-dev`, `FLUX.2-dev`) are gated on the Hub: accept the licence +on the model page with the `HF_TOKEN` account, then add an entry to `MODELS` in `local_gen.py` and `GENERATORS` in +`generators.py`. FLUX 3 *Image* has no public open weights at the time of writing (only FLUX 3 Action does). diff --git a/interface/__init__.py b/interface/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/interface/backend/__init__.py b/interface/backend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..25ad85bc41fe895319223ae74133b556aed7501f --- /dev/null +++ b/interface/backend/__init__.py @@ -0,0 +1 @@ +"""SynthScope web-interface backend package.""" diff --git a/interface/backend/app.py b/interface/backend/app.py new file mode 100644 index 0000000000000000000000000000000000000000..330e9acefd9ed9259327e641a0a6a82d08282674 --- /dev/null +++ b/interface/backend/app.py @@ -0,0 +1,522 @@ +"""FastAPI app for the SynthScope detector web interface. + +Endpoints: + GET /api/models -> list detectors + availability + GET /api/health -> liveness + device + POST /api/predict -> {file, model} -> fake/real probability + +Serves the built Vite frontend from ``frontend/dist`` if present, otherwise the no-build +standalone page in ``static/index.html`` (works without Node). +""" +import asyncio +import json +import tempfile +import threading +from pathlib import Path + +from fastapi import Body, FastAPI, File, Form, HTTPException, Query, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, Response +from fastapi.staticfiles import StaticFiles + +from . import config, datasets, media +from .engine import get_engine +from .registry import REPRO, REPRO_FULL, get_spec, input_note, list_specs + +app = FastAPI(title="SynthScope Detector Interface", version="1.0") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + +FRONTEND_DIST = config.INTERFACE_DIR / "frontend" / "dist" +STATIC_DIR = config.INTERFACE_DIR / "static" + + +@app.get("/api/health") +def health(): + return {"status": "ok", "device": config.device(), + "checkpoints_dir": str(config.CHECKPOINTS_DIR)} + + +_TRENDING_PATH = Path(__file__).resolve().parent / "trending.json" + + +@app.get("/api/trending") +def trending(): + """High-quality trending AIGC-detection papers (image + video). Refreshed daily by a + scheduled task that updates trending.json; TRENDING.md and the README teaser are regenerated from the same file.""" + try: + return json.loads(_TRENDING_PATH.read_text()) + except FileNotFoundError: + return {"updated": None, "image": [], "video": [], "landscape": [], "criteria": ""} + + +@app.get("/api/models") +def models(): + engine = get_engine() + out = [] + for spec in list_specs(): + out.append({ + "key": spec.key, + "name": spec.name, + "paper": spec.paper, + "modality": spec.modality, + "available": engine.is_available(spec), + "checkpoint": spec.checkpoint, + "weights": spec.weights, # "official" | "synthscope_trained" | "training_free" + "notes": spec.notes, + "input": input_note(spec), # {label, explanation}: what the detector does to the image + "repro": REPRO.get(spec.key), + "repro_full": REPRO_FULL.get(spec.key), + }) + return {"models": out, "device": config.device()} + + +@app.post("/api/predict") +async def predict(file: UploadFile = File(...), model: str = Form(...)): + try: + spec = get_spec(model) + except KeyError as e: + raise HTTPException(status_code=404, detail=str(e)) + + engine = get_engine() + if not engine.is_available(spec): + raise HTTPException( + status_code=409, + detail=f"No checkpoint available for '{spec.key}'. Expected file " + f"{spec.checkpoint} under {config.CHECKPOINTS_DIR}.", + ) + + data = await file.read() + if len(data) == 0: + raise HTTPException(status_code=400, detail="Empty upload.") + if len(data) > config.MAX_UPLOAD_BYTES: + raise HTTPException(status_code=413, detail="File too large.") + + try: + kind = media.media_kind(file.filename or "", file.content_type) + except ValueError as e: + raise HTTPException(status_code=415, detail=str(e)) + + try: + if kind == "image": + if spec.modality == "video": + raise HTTPException( + status_code=400, + detail=f"'{spec.name}' is a video detector; please upload a video.", + ) + result = engine.predict_image(spec, media.decode_image(data)) + result["media_type"] = "image" + return result + + # video upload -> write to a temp file for the decoder + suffix = Path(file.filename or "video.mp4").suffix or ".mp4" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as tmp: + tmp.write(data) + tmp.flush() + if spec.modality == "video": + result = engine.predict_video_clip(spec, tmp.name) + else: + frames = media.sample_video_pil(tmp.name, config.VIDEO_IMAGE_FRAMES) + result = engine.predict_video_frames(spec, frames) + result["media_type"] = "video" + return result + except HTTPException: + raise + except FileNotFoundError as e: + raise HTTPException(status_code=409, detail=str(e)) + except Exception as e: # surface model/decoding errors as 500 with a message + raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}") + + +# --- SOTA generator vs detector comparison demo ------------------------------ +# Pick a real image from our datasets -> send it to a current SOTA generator (Nano Banana / GPT Image / +# Grok / FLUX) with "make another image conceptually similar to this" -> score the generated image with +# the SynthScope detectors (best/promoted checkpoints) so the user sees how today's detectors hold up. +import base64 as _b64, glob as _glob, random as _random +from . import generators as _gens + +from .datasets import _root_candidates # node-portable scratch-mount remap (/hddstore<->/ssdstore<->...) + +# Base patterns point at cn023 /hddstore; expand each onto the other scratch mounts so the +# real-image pool resolves wherever the data was staged (cees /ssdstore, etc.). +_REAL_POOL_PATTERNS = [ + "/hddstore/ssalehi/SynthScope/GenImage/test/real", + "/hddstore/ssalehi/SynthScope/CNNDetection/test/*/0_real", + "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/*/0_real", +] +_REAL_POOLS = [c for p in _REAL_POOL_PATTERNS for c in _root_candidates(p)] + + +@app.get("/api/sota/generators") +def sota_generators(): + return {"generators": _gens.list_generators(), "default_prompt": _gens.DEFAULT_PROMPT, "device": config.device()} + + +@app.get("/api/sota/real_image") +def sota_real_image(): + """Return a random REAL image from our datasets (base64 PNG) for the comparison demo.""" + candidates = [] + for pat in _REAL_POOLS: + for d in _glob.glob(pat): + fs = _glob.glob(d + "/*") + if fs: + candidates.append(_random.choice(fs)); break + if len(candidates) >= 3: + break + if not candidates: + raise HTTPException(status_code=404, detail="No real images found in datasets.") + path = _random.choice(candidates) + img = media.decode_image(open(path, "rb").read()) + buf = __import__("io").BytesIO(); img.convert("RGB").save(buf, "PNG") + return {"image_b64": _b64.b64encode(buf.getvalue()).decode(), "source": path.split("/")[-3:]} + + +_GPU_LOCK = threading.Lock() # detector scoring and heavy local generation never overlap on the GPU + + +async def _sota_generate(real_bytes: bytes, generator: str, prompt: str, mode: str = "recreate"): + """Run one generator on the real image -> (effective prompt, generated PNG bytes); HTTP errors on failure.""" + if not real_bytes: + raise HTTPException(status_code=400, detail="Empty image.") + # text-to-image generators left on the old default prompt get a caption-derived one (generators.effective_prompt) + prompt = _gens.effective_prompt(generator, real_bytes, prompt) + try: + # off the event loop: a local generation takes 1-4 minutes and must not freeze the whole UI + heavy = generator in _gens.GENERATORS and _gens.GENERATORS[generator].heavy + + def run() -> bytes: + if not heavy: + return _gens.generate(generator, real_bytes, prompt, mode) + with _GPU_LOCK: # waits for any in-flight scoring to finish first + get_engine().release_gpu() + return _gens.generate(generator, real_bytes, prompt, mode) + return prompt, await asyncio.to_thread(run) + except KeyError as e: + raise HTTPException(status_code=404, detail=str(e)) + except RuntimeError as e: # missing API key + raise HTTPException(status_code=409, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=502, detail=f"generator failed: {type(e).__name__}: {e}") + + +def _score_one(engine, key: str, real_img, gen_img) -> dict: + try: + spec = get_spec(key) + if spec.modality != "image" or not engine.is_available(spec): + return {"detector": key, "error": "unavailable / not an image detector"} + rr, gr = engine.predict_image(spec, real_img), engine.predict_image(spec, gen_img) + return {"detector": key, "name": spec.name, + "real_fake_prob": round(rr["fake_probability"], 4), + "gen_fake_prob": round(gr["fake_probability"], 4), + "gen_verdict": gr.get("verdict"), "caught": gr["fake_probability"] >= 0.5} + except Exception as e: + return {"detector": key, "error": f"{type(e).__name__}: {e}"} + + +def _sota_score(real_bytes: bytes, gen_bytes: bytes, detectors: str) -> list[dict]: + """Score the real and the generated image with each listed detector (no generation involved).""" + real_img, gen_img = media.decode_image(real_bytes), media.decode_image(gen_bytes) + engine = get_engine() + results = [] + for key in [d.strip() for d in detectors.split(",") if d.strip()]: + with _GPU_LOCK: # one detector at a time: concurrent requests queue, not race + results.append(_score_one(engine, key, real_img, gen_img)) + return results + + +def _layout_similarity(a_bytes: bytes, b_bytes: bytes) -> float: + """Correlation of the two images' 64x64 grayscale layouts, -1..1. About 0.9+ means the generated image is an edit + of the source (same composition); low values mean a genuinely new image.""" + import numpy as np + def thumb(b: bytes): + im = media.decode_image(b).convert("L").resize((64, 64)) + return np.asarray(im, dtype="float64") - np.asarray(im, dtype="float64").mean() + a, b = thumb(a_bytes), thumb(b_bytes) + return float((a * b).sum() / (((a * a).sum() * (b * b).sum()) ** 0.5 + 1e-8)) + + +@app.post("/api/sota/describe") +async def sota_describe(file: UploadFile = File(...)): + """Caption the source photo and return a text-to-image prompt for it (the user can edit it before generating).""" + data = await file.read() + if not data: + raise HTTPException(status_code=400, detail="Empty image.") + return await asyncio.to_thread(_gens.describe_prompt, data) + + +@app.post("/api/sota/generate") +async def sota_generate(file: UploadFile = File(...), generator: str = Form(...), + prompt: str = Form(_gens.DEFAULT_PROMPT), mode: str = Form("recreate")): + """real image + generator -> the generated image only. Scoring is a separate call (/api/sota/score), so + trying another detector never regenerates. mode "recreate" builds a new image from the prompt alone, + "edit" edits the source photo.""" + real_bytes = await file.read() + prompt, gen_bytes = await _sota_generate(real_bytes, generator, prompt, mode) + sim = _layout_similarity(real_bytes, gen_bytes) + return {"generator": generator, "mode": mode, "prompt": prompt, "similarity": round(sim, 3), + "near_copy": sim >= 0.8, "real_size": list(media.decode_image(real_bytes).size), + "gen_size": list(media.decode_image(gen_bytes).size), "generated_b64": _b64.b64encode(gen_bytes).decode()} + + +@app.post("/api/sota/score") +async def sota_score(real: UploadFile = File(...), generated: UploadFile = File(...), detectors: str = Form(...)): + """Score an existing real/generated pair with the listed detectors (fast: no generation).""" + real_bytes, gen_bytes = await real.read(), await generated.read() + if not real_bytes or not gen_bytes: + raise HTTPException(status_code=400, detail="Empty image.") + return {"results": await asyncio.to_thread(_sota_score, real_bytes, gen_bytes, detectors)} + + +@app.post("/api/sota/compare") +async def sota_compare(file: UploadFile = File(...), generator: str = Form(...), + detectors: str = Form(...), prompt: str = Form(_gens.DEFAULT_PROMPT)): + """Generate + score in one call (kept for API clients; the web UI uses generate then score).""" + real_bytes = await file.read() + prompt, gen_bytes = await _sota_generate(real_bytes, generator, prompt, "edit") + return {"generator": generator, "prompt": prompt, "generated_b64": _b64.b64encode(gen_bytes).decode(), + "results": await asyncio.to_thread(_sota_score, real_bytes, gen_bytes, detectors)} + + +# --- Leaderboard: all methods x benchmarks (paper vs SynthScope-trained vs official), by benchmark ------ +_METHOD_NAMES = { + "clip": "CLIP / UnivFD", "cnn": "CNNDetection", "fat_former": "FatFormer", "npr": "NPR", + "aide": "AIDE", "safe": "SAFE", "co_spy": "Co-Spy", "spai": "SPAI", "vib_net": "VIB-Net", + "pgc": "PGC", "cure": "CuRe", "d3": "D3", "restrav": "ReStraV", "stall": "STALL", +} +_FAMILY = [ # (substring -> benchmark family label); first match wins + ("genimage", "GenImage"), ("forensynth", "ForenSynths"), ("universalfakedetect", "UniversalFakeDetect"), + ("ojha", "UniversalFakeDetect"), ("ufd", "UniversalFakeDetect"), ("aigcdetect", "AIGCDetect"), + ("co-spy-bench", "Co-Spy-Bench"), ("aigibench", "AIGIBench"), ("aigi", "AIGIBench"), + ("commgen", "CommGen15"), ("genvideo", "GenVideo"), ("evalcrafter", "EvalCrafter"), + ("videophy", "VideoPhy"), ("vidprom", "VidProM"), ("genvidbench", "GenVidBench"), + ("ditfake", "DiTFake"), ("diffusionforensics", "DiffusionForensics"), + ("diffusion1kstep", "Diffusion1kStep"), # before self-synthesis (NPR Table 5 contains both) + ("gangen", "GANGen / Self-Synthesis"), ("self-synthesis", "GANGen / Self-Synthesis"), + ("chameleon", "Chameleon"), ("synthbuster", "Synthbuster"), ("in-the-wild", "In-the-wild"), + ("videofeedback", "VideoFeedback"), ("comgenvid", "ComGenVid"), + ("physics-iq", "Physics-IQ"), ("veo3", "Veo3 zero-shot"), ("demamba", "DeMamba"), + ("gan-source", "GAN-source"), ("diffusion-source", "Diffusion-source"), +] + + +def _family(name: str) -> str: + n = (name or "").lower() + for sub, fam in _FAMILY: + if sub in n: + return fam + return (name.split("(")[0].strip() or name)[:28] + + +def _lead_num(s: str) -> float: + import re as _re + m = _re.search(r"(\d+\.?\d*)", s or "") + return float(m.group(1)) if m else -1.0 + + +@app.get("/api/leaderboard") +def leaderboard(): + """Every method x benchmark with paper / SynthScope-trained / official-weights numbers, grouped by + benchmark family so methods that share a benchmark can be compared head-to-head.""" + groups: dict[str, list] = {} + for key, rows in REPRO_FULL.items(): + for r in rows: + fam = _family(r["benchmark"]) + groups.setdefault(fam, []).append({ + "method": _METHOD_NAMES.get(key, key), "key": key, + "benchmark": r["benchmark"], "paper": r["paper"], + "synthscope": r.get("ours_synthscope", ""), "official": r.get("ours_official", ""), + "status": r["status"], + "_sort": max(_lead_num(r.get("ours_synthscope")), _lead_num(r.get("ours_official"))), + }) + out = [] + for fam, methods in groups.items(): + methods.sort(key=lambda m: m["_sort"], reverse=True) + for m in methods: + m.pop("_sort", None) + out.append({"benchmark": fam, "n_methods": len(methods), "rows": methods}) + out.sort(key=lambda g: (-g["n_methods"], g["benchmark"])) # most cross-method-compared first + rows = [r for g in out for r in g["rows"]] + shown = {r["key"] for r in rows} + summary = { + "benchmarks": len(out), "results": len(rows), "detectors_listed": len(shown), + "detectors_total": len(list_specs()), + "by_status": {st: sum(1 for r in rows if r["status"] == st) + for st in ("reproduced", "official", "below", "reference", "missing")}, + "not_listed": [sp.name for sp in list_specs() if sp.key not in shown], + } + return {"groups": out, "summary": summary} + + +def _generator_runs() -> list[Path]: + """Result folders: benchmark/generator_runs// (made by benchmark/generator_bench.py; names starting with + '_' or the 'reference' prompt set are skipped) plus the legacy repro/ideogram_bench* runs.""" + import glob as _glob2 + runs = sorted(Path(d) for d in _glob2.glob(str(config.REPO_ROOT / "benchmark" / "generator_runs" / "*")) + if not Path(d).name.startswith("_") and Path(d).name != "reference") + runs += sorted(Path(d) for d in _glob2.glob(str(config.REPO_ROOT / "repro" / "ideogram_bench*"))) + return [d for d in runs if (d / "results.json").is_file()] + + +@app.get("/api/generator_benchmarks") +def generator_benchmarks(): + """Detector-vs-generator benchmarks, one card per generator plus an overview matrix (detector x generator AUROC). + Add a generator by running benchmark/generator_bench.py (see docs/BENCHMARKS.md).""" + import json as _json + out = [] + for d in _generator_runs(): + rows = _json.loads((d / "results.json").read_text()) + man_f = d / "manifest.json" + meta = _json.loads(man_f.read_text()) if man_f.is_file() else {} + n = meta.get("n") or len(meta.get("pairs", [])) or None + out.append({ + "generator": meta.get("label") or "Ideogram 4.0", + "key": d.name, "n_real": n, "n_fake": n, "res": meta.get("res", 0), + "settings": meta.get("settings") or f"{meta.get('res','?')}px · {meta.get('steps','?')} steps", + "note": "Real photos (people excluded) against images the generator created from a caption of each photo, text " + "only. Detectors are the SynthScope checkpoints with their paper-faithful preprocessing. None were " + "trained on this generator, so this measures generalization. Ranked by AUROC.", + "rows": rows, + }) + out.sort(key=lambda b: (b["generator"], -b.get("res", 0))) + labels = [b["generator"] for b in out] + names = sorted({r["name"] for b in out for r in b["rows"]}) + table = [] + for nm in names: + vals = [next((r["auroc"] for r in b["rows"] if r["name"] == nm), None) for b in out] + got = [v for v in vals if v is not None] + table.append({"name": nm, "auroc": vals, "mean": round(sum(got) / len(got), 2) if got else None}) + table.sort(key=lambda r: -(r["mean"] or 0)) + return {"benchmarks": out, "overview": {"generators": labels, "rows": table}} + + +# --- dataset explorer -------------------------------------------------------- +@app.get("/api/datasets") +def datasets_list(): + """Datasets present on disk + which dataset each detector was trained on.""" + return { + "datasets": datasets.list_datasets(), + "model_train": datasets.MODEL_TRAIN_DATASET, + } + + +@app.get("/api/datasets/{key}/sample") +def dataset_sample(key: str, split: str = "test", n: int = 24, seed: int = 0): + try: + items = datasets.sample_batch(key, split, n=max(1, min(n, 60)), seed=seed) + except KeyError as e: + raise HTTPException(status_code=404, detail=str(e)) + except FileNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + return {"dataset": key, "split": split, "seed": seed, "items": items} + + +@app.get("/api/datasets/{key}/media") +def dataset_media(key: str, rel: str = Query(...), thumb: int = 0): + try: + path = datasets.resolve_media(key, rel) + except KeyError as e: + raise HTTPException(status_code=404, detail=str(e)) + except PermissionError as e: + raise HTTPException(status_code=403, detail=str(e)) + except FileNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + if thumb: + modality = datasets.get_def(key).modality + try: + return Response(content=media.thumbnail_bytes(path, modality), media_type="image/jpeg") + except Exception as e: + raise HTTPException(status_code=500, detail=f"thumbnail failed: {e}") + return FileResponse(str(path)) + + +def _run_on_dataset_item(engine, spec, dataset: str, rel: str) -> dict: + """Run ``spec`` on an on-disk dataset file ``rel`` and attach ground-truth/correctness. + + Raises HTTPException for caller-facing errors. Shared by /api/predict_dataset (single item, + full result) and /api/score_batch (many items, scores only). + """ + path = datasets.resolve_media(dataset, rel) # may raise Permission/FileNotFound + d = datasets.get_def(dataset) + if d.modality == "video": + if spec.modality == "video": + result = engine.predict_video_clip(spec, str(path)) + else: + frames = media.sample_video_pil(str(path), config.VIDEO_IMAGE_FRAMES) + result = engine.predict_video_frames(spec, frames) + result["media_type"] = "video" + else: + if spec.modality == "video": + raise HTTPException(status_code=400, detail=f"'{spec.name}' is a video detector; pick a video dataset.") + with open(path, "rb") as fh: + result = engine.predict_image(spec, media.decode_image(fh.read())) + result["media_type"] = "image" + gt = datasets.label_of(Path(rel)) + result["ground_truth"] = gt + result["correct"] = (gt is not None and result["verdict"] == gt) + return result + + +@app.post("/api/predict_dataset") +def predict_dataset(dataset: str = Form(...), rel: str = Form(...), model: str = Form(...)): + """Run a detector on a dataset file already on disk (no re-upload) + return ground truth.""" + try: + spec = get_spec(model) + except KeyError as e: + raise HTTPException(status_code=404, detail=str(e)) + engine = get_engine() + if not engine.is_available(spec): + raise HTTPException(status_code=409, detail=f"No checkpoint available for '{spec.key}'.") + try: + return _run_on_dataset_item(engine, spec, dataset, rel) + except HTTPException: + raise + except (KeyError, PermissionError, FileNotFoundError) as e: + raise HTTPException(status_code=404, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}") + + +@app.post("/api/score_batch") +def score_batch(payload: dict = Body(...)): + """Score many dataset items at once (for threshold calibration). Body: {dataset, model, rels:[...]}. + + Returns ``{items:[{rel, fake_probability, ground_truth}]}`` -- per-item errors are reported inline + (``error`` field) instead of failing the whole batch, so calibration uses whatever decoded.""" + dataset = payload.get("dataset"); model = payload.get("model"); rels = payload.get("rels") or [] + try: + spec = get_spec(model) + except KeyError as e: + raise HTTPException(status_code=404, detail=str(e)) + engine = get_engine() + if not engine.is_available(spec): + raise HTTPException(status_code=409, detail=f"No checkpoint available for '{spec.key}'.") + out = [] + for rel in rels[:200]: + try: + r = _run_on_dataset_item(engine, spec, dataset, rel) + out.append({"rel": rel, "fake_probability": r["fake_probability"], "ground_truth": r["ground_truth"]}) + except Exception as e: + out.append({"rel": rel, "error": f"{type(e).__name__}: {getattr(e, 'detail', e)}"}) + return {"dataset": dataset, "model": model, "items": out} + + +# --- static / SPA serving ---------------------------------------------------- +if FRONTEND_DIST.is_dir(): + app.mount("/assets", StaticFiles(directory=str(FRONTEND_DIST / "assets")), name="assets") + +# Serve the standalone page's local assets (vendored React/htm) so the UI needs no external CDN. +app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") + + +_NO_CACHE = {"Cache-Control": "no-cache, no-store, must-revalidate", "Pragma": "no-cache", "Expires": "0"} + + +@app.get("/") +def index(): + # Serve the UI with no-store so a plain reload always picks up edits (no stale cached page). + src = FRONTEND_DIST / "index.html" if (FRONTEND_DIST / "index.html").is_file() else STATIC_DIR / "index.html" + return FileResponse(str(src), headers=_NO_CACHE) diff --git a/interface/backend/build_repro_full.py b/interface/backend/build_repro_full.py new file mode 100644 index 0000000000000000000000000000000000000000..7932a1bd6810cb413195eae49744bef925a40dbd --- /dev/null +++ b/interface/backend/build_repro_full.py @@ -0,0 +1,217 @@ +"""Build interface/backend/repro_full.json: per-method list of ALL reported benchmarks with +SEPARATE columns for the SynthScope-trained checkpoint and the official-weights-in-SynthScope-class number. + +Source of truth = the verified specs in repro/benchmark_specs/*.json. Numbers that the specs only +record in prose notes (the AGID/official split is structured inconsistently across methods) are +pulled into an explicit, auditable VAL table below — every value here traces to a file under +repro/bench_results/ (see the per-row tags in the spec notes). Run with --audit to print the table. +""" +import json, glob, os, sys +from pathlib import Path + +ROOT = str(Path(__file__).resolve().parents[2]) # the repository root, wherever it is checked out + +# (model, benchmark-name-substring) -> verified {synthscope:{...}, official:{...}} numbers. +# These resolve the SynthScope-vs-official ambiguity in the spec our_number dicts and surface the number +# that the spec only kept in prose. Each traces to a repro/bench_results/*.json (tag in spec note). +VAL = { + # AIGCDetect is the ProGAN-trained setting (B1); we only retrained the SDv1.4 setting, so no SynthScope_train here. + ("aide", "AIGCDetect"): {"official": {"map": 97.94, "acc": 91.89}}, + # SynthScope_train = faithful native-res 5-view retrain (aide_native_v2/model_2) — reproduces official. + ("aide", "GenImage"): {"official": {"map": 97.12, "acc": 85.91}, "synthscope": {"map": 96.42, "acc": 77.87}}, + ("clip", "ForenSynths"): {"synthscope": {"map": 97.22, "acc": 88.28}, "official": {"map": 96.95}}, + ("clip", "UniversalFakeDetect"): {"synthscope": {"map": 96.44, "acc": 87.23}, "official": {"map": 95.66}}, + ("cnn", "ForenSynths"): {"synthscope": {"map": 94.42, "acc": 76.36}, "official": {"map": 92.67}}, + ("co_spy", "Co-Spy-Bench"): {"synthscope": {"map": 96.6, "acc": 89.9}, "official": {"map": 86.65, "acc": 83.88}}, + ("co_spy", "AIGCDetect"): {"official": {"map": 94.59, "acc": 89.25}, "synthscope": {"map": 84.91}}, + ("co_spy", "GenImage"): {"official": {"map": 92.41}, "synthscope": {"map": 90.59, "acc": 75.91}}, + ("co_spy", "ForenSynths"): {"official": {"map": 87.97}, "synthscope": {"map": 96.17}}, + # CuRe: only the official adapter exists (the training recipe is unreleased), so there is no SynthScope-trained number. + ("cure", "AIGIBench"): {"official": {"map": 97.0, "acc": 92.85, "auroc": 97.48}}, + ("cure", "UniversalFakeDetect"): {"official": {"map": 99.19, "acc": 93.94}}, + ("cure", "ForenSynths"): {"official": {"map": 94.05, "acc": 81.95}}, + ("cure", "GenImage"): {"official": {"map": 97.3, "acc": 85.18}}, + ("fat_former", "4-class"): {"official": {"map": 99.7, "acc": 98.39}}, + ("fat_former", "Diffusion-model"): {"official": {"map": 98.6, "acc": 93.72}}, + ("npr", "Table 1"): {"official": {"map": 94.22, "acc": 92.30}}, + ("npr", "Table 2"): {"official": {"map": 96.49, "acc": 94.81}}, + ("npr", "Table 3"): {"official": {"map": 99.4, "acc": 95.03}}, + ("npr", "Table 4"): {"official": {"map": 96.69, "acc": 93.94}}, + ("npr", "Table 5"): {"official": {"map": 83.47, "acc": 79.13}}, + # SAFE now SERVES the SynthScope-trained checkpoint (retrain v2 / model_38), which reproduces the paper. + ("safe", "ForenSynths"): {"official": {"map": 98.78, "acc": 96.2}, "synthscope": {"map": 97.79, "acc": 95.1}}, + ("safe", "GANGen"): {"official": {"map": 99.85, "acc": 99.0}}, + ("safe", "UniversalFakeDetect"): {"official": {"map": 99.1, "acc": 95.71}, "synthscope": {"map": 98.44, "acc": 94.0}}, + ("safe", "GenImage"): {"official": {"map": 99.32, "acc": 95.62}, "synthscope": {"map": 99.51, "acc": 95.9}}, + ("safe", "DiTFake"): {"synthscope": {"map": 99.96, "acc": 99.2}}, + ("safe", "DiffusionForensics"): {"synthscope": {"map": 99.34, "acc": 91.3}}, + ("pgc", "GenImage"): {"official": {"map": 100.0, "acc": 99.49}, "synthscope": {"map": 97.62, "acc": 78.6}}, + ("pgc", "UniversalFakeDetect"): {"official": {"map": 99.34, "acc": 94.16}, "synthscope": {"map": 99.01}}, + ("pgc", "ForenSynths"): {"official": {"map": 96.25, "acc": 83.73}, "synthscope": {"map": 89.03}}, + ("pgc", "AIGIBench"): {"official": {"map": 95.36, "acc": 88.33}, "synthscope": {"map": 86.88}}, + ("pgc", "CommGen15"): {"official": {"map": 99.26, "acc": 97.39}, "synthscope": {"map": 98.61, "acc": 66.66}}, + # SPAI now SERVES official weights remapped into the SynthScope class, scored at NATIVE resolution + # (the faithful any-res path). official = that served number; synthscope = the superseded from-scratch retrain. + ("spai", "GenImage"): {"official": {"map": 94.22, "auroc": 94.28}, "synthscope": {"map": 88.8, "acc": 82.81}}, + ("spai", "Synthbuster"): {"official": {"map": 96.75, "auroc": 96.97}}, + # D3 now uses the faithful protocol (X-CLIP-B/16 + 10% edge-crop). On GenVideo, AUROC/acc are the + # robust metrics (the mAP harness is degenerate: 10000 reals : as few as 56 fakes/generator). + ("d3", "GenVideo"): {"synthscope": {"map": 63.93, "auroc": 95.95, "acc": 91.71}}, + ("d3", "EvalCrafter"): {"synthscope": {"map": 91.19, "auroc": 95.41, "acc": 87.67}}, + ("restrav", "GenVidBench"): {"synthscope": {"map": 92.7, "auroc": 95.9, "acc": 86.87}}, + ("stall", "GenVideo"): {"synthscope": {"map": 82.38, "auroc": 83.51, "acc": 56.88}}, + ("stall", "VideoFeedback"): {"synthscope": {"map": 83.07, "auroc": 85.1, "acc": 62.12}}, + ("stall", "ComGenVid"): {"synthscope": {"map": 82.59, "auroc": 84.0, "acc": 57.34}}, + ("vib_net", "Diffusion-source (SDv1.4-trained), AP"): {"synthscope": {"map": 83.94, "acc": 77.57}}, + ("vib_net", "Diffusion-source (SDv1.4-trained), ACC"): {"synthscope": {"map": 83.94, "acc": 77.57}}, + ("vib_net", "GAN-source (ForenSynths/ProGAN-trained), AP"): {"synthscope": {"map": 61.71, "acc": 57.91}}, + ("vib_net", "GAN-source (ForenSynths/ProGAN-trained), ACC"): {"synthscope": {"map": 61.71, "acc": 57.91}}, + ("co_spy", "In-the-wild"): {}, # genuinely not evaluated (prevents the "Co-Spy-Bench" substring collision) +} + +# Tolerance (percentage points) for calling a number a reproduction of the paper. +TOL = {"map": 2.0, "acc": 2.0, "auroc": 2.5} + +LAB = {"map": "mAP", "acc": "acc", "auroc": "AUROC"} + + +def paper_metrics(d): + """Normalize a paper_number dict to {map,acc,auroc} in PERCENT (fractions like 0.80 -> 80).""" + out = {} + if not isinstance(d, dict): + return out + for k, v in d.items(): + if not isinstance(v, (int, float)): + continue + kl = k.lower() + val = v * 100 if 0 < v <= 1.0 else v + if "auc" in kl or "auroc" in kl: + out.setdefault("auroc", val) + elif kl == "acc" or kl.startswith("acc") or "_acc" in kl: + out.setdefault("acc", val) + elif "ap" in kl or "map" in kl: + out.setdefault("map", val) + return out + + +def cmp_ok(ours, pm): + """True/False if ours reproduces the paper on every shared metric; None if not comparable.""" + shared = [m for m in ("map", "acc", "auroc") if m in ours and m in pm] + if not shared: + return None + return all(ours[m] >= pm[m] - TOL[m] for m in shared) + + +def fmt_paper(d): + if not isinstance(d, dict): + return "—" if d in (None, "") else str(d) + parts = [f"{k}={v}" for k, v in d.items() + if k != "note" and v not in (None, "") and not isinstance(v, (dict, list))] + return " / ".join(parts) if parts else "—" + + +def has_paper_number(d): + if not isinstance(d, dict): + return d not in (None, "") + return any(k != "note" and v not in (None, "") and not isinstance(v, (dict, list)) + for k, v in d.items()) + + +def fmt_ours(d): + if not d: + return "" + return " / ".join(f"{LAB[k]}={d[k]}" for k in ("map", "acc", "auroc") if k in d) + + +def combined(synthscope, official): + a, o = fmt_ours(synthscope), fmt_ours(official) + if a and o: + return f"SynthScope {a} · official {o}" + if o: + return f"{o} (official wts)" + return a + + +def match_val(model, name): + """Return (synthscope, official, matched_substr) for the best VAL hit. A prefix match wins over a + mid-string match (so 'In-the-wild (Co-Spy-Bench ...)' matches the In-the-wild entry, not Co-Spy-Bench).""" + best = None # (is_prefix, len, sub, v) + nl = name.lower() + for (m, sub), v in VAL.items(): + if m == model and sub.lower() in nl: + cand = (nl.startswith(sub.lower()), len(sub), sub, v) + if best is None or cand[:2] > best[:2]: + best = cand + if not best: + return {}, {}, None + return best[3].get("synthscope", {}), best[3].get("official", {}), best[2] + + +def status_for(name, paper, synthscope, official, spec_status): + """reproduced = SynthScope-trained matches paper; official = official-weights-in-SynthScope matches paper + (SynthScope-trained number shown alongside where attempted); below = neither; reference = paper gives + no number for this cross-dataset; missing = not evaluated. Falls back to the spec's label only + when the paper metric keys are not numerically comparable (e.g. ReStraV's nested per-split dicts).""" + if not has_paper_number(paper): + return "reference" if (synthscope or official) else "missing" + if not (synthscope or official): + return "missing" + pm = paper_metrics(paper) + if not pm: # paper has numbers but not in comparable keys + return spec_status or "below" + synthscope_ok, off_ok = cmp_ok(synthscope, pm), cmp_ok(official, pm) + if synthscope_ok: + return "reproduced" + if off_ok: + return "official" + if synthscope_ok is None and off_ok is None: # neither comparable -> trust the spec + return spec_status or "below" + return "below" + + +out = {} +audit = [] +for f in sorted(glob.glob(f"{ROOT}/repro/benchmark_specs/*.json")): + d = json.load(open(f)) + model = d.get("model") + rows = [] + for b in d.get("benchmarks", []): + name = b.get("name", "") or "" + paper = b.get("paper_number") + spec_status = b.get("our_status", "") + synthscope, official, sub = match_val(model, name) + status = status_for(name, paper, synthscope, official, spec_status) + on = b.get("our_number") + note = "" + if isinstance(on, dict) and on.get("note"): + note = on["note"] + else: + note = (b.get("gap_cause") or b.get("dataset_source") or "") + rows.append({ + "benchmark": name[:46], + "metric": b.get("metric", ""), + "paper": fmt_paper(paper), + "ours": combined(synthscope, official), + "ours_synthscope": fmt_ours(synthscope), + "ours_official": fmt_ours(official), + "status": status, + "note": note[:400], + }) + audit.append((model, name[:38], fmt_paper(paper)[:22], + (fmt_ours(synthscope) or "-")[:22], (fmt_ours(official) or "-")[:22], + f"{spec_status}->{status}")) + out[model] = rows + +json.dump(out, open(f"{ROOT}/interface/backend/repro_full.json", "w"), indent=2) +n = sum(len(v) for v in out.values()) +counts = {} +for v in out.values(): + for r in v: + counts[r["status"]] = counts.get(r["status"], 0) + 1 +print(f"wrote repro_full.json: {len(out)} methods, {n} rows status={counts}") + +if "--audit" in sys.argv: + print(f"\n{'model':10} {'benchmark':38} {'paper':22} {'SynthScope':22} {'official':22} status") + print("-" * 140) + for r in audit: + print(f"{r[0]:10} {r[1]:38} {r[2]:22} {r[3]:22} {r[4]:22} {r[5]}") diff --git a/interface/backend/config.py b/interface/backend/config.py new file mode 100644 index 0000000000000000000000000000000000000000..cd089d0f2d949a4bc65107bdcc724d1ddeeb2e1c --- /dev/null +++ b/interface/backend/config.py @@ -0,0 +1,101 @@ +"""Runtime configuration and path wiring for the SynthScope web interface. + +The interface imports the SynthScope repo (models, dataset decoders, Hydra config composition) +directly, so we add the repo root to ``sys.path`` and pin a few cache locations that make +sense on the GPU node where the models/checkpoints actually live (cn023 -> /hddstore). +Everything is overridable via environment variables. +""" +import os +import sys +from pathlib import Path + +import _synthscope_env # noqa: F401 (maps legacy AGID_* variables to SYNTHSCOPE_*) + +# interface/backend/config.py -> interface/ -> repo root +BACKEND_DIR = Path(__file__).resolve().parent +INTERFACE_DIR = BACKEND_DIR.parent +REPO_ROOT = INTERFACE_DIR.parent + +# Make the SynthScope package importable (models.*, dataset, config_utils, ...). +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +def _strip_env_value(v: str) -> str: + """Parse a ``.env`` value: honor quotes, drop an inline ``# comment``. + + Tolerates the common mistake of filling a key in next to the template's trailing comment, e.g. + ``OPENAI_API_KEY=sk-... # GPT Image -> platform.openai.com/api-keys`` — without this, the comment + becomes part of the key and the API rejects it. A ``#`` only starts a comment when it is at the + start or preceded by whitespace, so keys/values that legitimately contain ``#`` are preserved. + """ + v = v.strip() + if v[:1] in ('"', "'"): # quoted: take quoted span, ignore trailing comment + q = v[0] + end = v.find(q, 1) + return v[1:end] if end != -1 else v[1:] + for i, ch in enumerate(v): # unquoted: cut at the first whitespace-led '#' + if ch == "#" and (i == 0 or v[i - 1] in " \t"): + v = v[:i] + break + return v.strip() + + +def _load_dotenv() -> None: + """Load secrets (SOTA-generator API keys, etc.) from a gitignored ``.env`` at the repo root. + + No dependency required: a minimal KEY=VALUE parser. Existing real env vars take precedence + (``setdefault``), so a shell export or a secrets manager always overrides the file. The ``.env`` + file is gitignored — never commit keys; commit only ``.env.example``. + """ + for env_path in (REPO_ROOT / ".env", INTERFACE_DIR / ".env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + val = _strip_env_value(v) + if val: # skip empty values (unfilled template keys) + os.environ.setdefault(k.strip(), val) + + +_load_dotenv() + +# Central checkpoint directory. Defaults to the portable ``~/.cache/synthscope`` home (matching the +# ``synthscope`` package + README); cluster jobs override with SYNTHSCOPE_CKPT_DIR for node-local scratch. +_LEGACY_HOME = Path.home() / ".cache" / "agid" # pre-rename location, reused if it exists +_HOME = Path(os.environ.get("SYNTHSCOPE_HOME") or (_LEGACY_HOME if _LEGACY_HOME.exists() else Path.home() / ".cache" / "synthscope")).expanduser() +CHECKPOINTS_DIR = Path( + os.environ.get("SYNTHSCOPE_CKPT_DIR", str(_HOME / "checkpoints")) +).expanduser() + +# Where to cache downloaded pretrained backbones (CLIP/DINOv2/SigLIP/ConvNeXt/...). On cn023 +# the /nvmestore caches referenced by the experiment scripts are not present, so default to a +# node-local cache. Only set if the caller has not already configured these. +_DEFAULT_CACHE = Path(os.environ.get("SYNTHSCOPE_CACHE_DIR", str(_HOME / "cache"))).expanduser() +os.environ.setdefault("HF_HOME", str(_DEFAULT_CACHE / "huggingface")) +os.environ.setdefault("TORCH_HOME", str(_DEFAULT_CACHE / "torch")) +os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "0") + +# Upload / decoding limits. +MAX_UPLOAD_BYTES = int(os.environ.get("SYNTHSCOPE_MAX_UPLOAD_MB", "200")) * 1024 * 1024 +# Number of frames sampled from a video when running an *image* detector frame-by-frame. +VIDEO_IMAGE_FRAMES = int(os.environ.get("SYNTHSCOPE_VIDEO_IMAGE_FRAMES", "16")) + +# How many models to keep resident at once (LRU). Big backbones (ConvNeXt-XXL, SigLIP+VAE) +# are several GB each, so we evict rather than risk OOM on a shared GPU. +MAX_RESIDENT_MODELS = int(os.environ.get("SYNTHSCOPE_MAX_RESIDENT_MODELS", "2")) + + +def device() -> str: + import torch + + return "cuda" if torch.cuda.is_available() else "cpu" + + +# xFormers has no CPU kernels: the DINOv2 hub code would fail on a CPU-only machine where xFormers is installed. +# Its own switch makes it fall back to standard attention. +if device() == "cpu": + os.environ.setdefault("XFORMERS_DISABLED", "1") diff --git a/interface/backend/datasets.py b/interface/backend/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..8b32422641d9b09f7814df20be37c4a5f7bb2728 --- /dev/null +++ b/interface/backend/datasets.py @@ -0,0 +1,297 @@ +"""Dataset explorer backend: discover on-disk datasets, sample random batches, serve +thumbnails, and report which dataset each detector was trained on. + +The dataset *configs* (conf/dataset/*.yaml) point at paths that are partly stale on the GPU +node (the /nvmestore roots are not mounted on cn023, and the video sets live in ad-hoc repro +dirs). So this module keeps an explicit registry of candidate roots and auto-detects which +ones actually exist, rather than trusting the configs. Labels are inferred from path +components (``0_real``/``real``/``nature`` -> real, ``1_fake``/``fake``/``ai`` -> fake), which +covers every on-disk layout (ForenSynths ``/{0_real,1_fake}``, GenImage ``{ai,nature}``, +GenVideo ``Real`` + ``Fake/``, ReStraV ``genvideo/{0_real,1_fake}``). +""" +from __future__ import annotations + +import os +import _synthscope_env # noqa: F401 (maps legacy AGID_* variables to SYNTHSCOPE_*) +import random +from dataclasses import dataclass, field +from pathlib import Path + +IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"} +VIDEO_EXTS = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".m4v"} + +# Directory-name tokens that identify a sample's ground-truth label. +REAL_TOKENS = {"0_real", "real", "reals", "nature", "natural", "coco", "imagenet"} +FAKE_TOKENS = {"1_fake", "fake", "fakes", "ai", "generated", "gen", "synthesis"} + +# How many paths to keep in the in-memory pool per label (reservoir-sampled over the full tree) +# and the hard cap on tree entries scanned, so sampling stays fast on 100k+ -image datasets. +POOL_PER_LABEL = 4000 +SCAN_CAP = 400_000 + + +@dataclass +class DatasetDef: + key: str + name: str + modality: str # "image" | "video" + roots: list[str] # candidate roots; first existing one wins + splits: list[str] = field(default_factory=lambda: ["train", "val", "test"]) + note: str = "" + + +# Explicit registry (auto-detected against disk at request time). +DATASET_DEFS: list[DatasetDef] = [ + DatasetDef("cnndetection", "CNNDetection / ForenSynths (ProGAN)", "image", + ["/hddstore/ssalehi/SynthScope/CNNDetection"], ["train", "val", "test"], + "ProGAN-family GAN images; the ForenSynths benchmark."), + DatasetDef("genimage", "GenImage (SD-v1.4)", "image", + ["/hddstore/ssalehi/SynthScope/GenImage"], ["train", "test"], + "ImageNet-class diffusion (SD-v1.4) vs real ImageNet."), + DatasetDef("cospy", "CoSpyBench", "image", + ["/hddstore/ssalehi/SynthScope/CoSpyBench"], ["test"], + "Co-Spy cross-generator evaluation benchmark."), + DatasetDef("cospy_subset", "Co-Spy ProGAN subset", "image", + ["/hddstore/ssalehi/SynthScope/spai_train/cospy_subset"], ["train", "val", "test"]), + DatasetDef("spai_ldm", "SPAI LDM trainset (vs COCO)", "image", + ["/hddstore/ssalehi/SynthScope/spai_train/staged"], ["train", "val", "test"], + "Latent-diffusion fakes vs COCO reals (SPAI from-scratch)."), + DatasetDef("genvideo_val", "GenVideo-Val", "video", + ["/hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val"], ["."], + "GenVideo validation pool (msrvtt reals + 10 generators)."), + DatasetDef("restrav_data", "ReStraV data (VidProM → GenVideo split)", "video", + ["/hddstore/ssalehi/SynthScope/restrav_data"], ["train", "val", "test"], + "Train = VidProM/DeMamba (VSC reals); test = GenVideo (msrvtt reals)."), + DatasetDef("ditfake", "DiTFake (DiT/FLUX/SD3 images)", "image", + ["/hddstore/ssalehi/SynthScope/DiTFake"], ["test"], + "DiT/FLUX/SD3-generated images vs COCO reals."), + DatasetDef("aigibench", "AIGIBench (multi-generator)", "image", + ["/hddstore/ssalehi/SynthScope/AIGIBench"], ["test"], + "25-generator AI-image benchmark (GAN/diffusion/face-swap) vs real."), + DatasetDef("evalcrafter", "EvalCrafter (T2V, generated)", "video", + ["/hddstore/ssalehi/SynthScope/EvalCrafter"], ["test"], + "14 text-to-video generators — all AI-generated (no reals)."), + DatasetDef("genvidbench", "GenVidBench", "video", + ["/hddstore/ssalehi/SynthScope/GenVidBench"], ["test"], + "Real (Vript, HD-VG-130M) vs 8 video generators."), + DatasetDef("freqnetdetection", "FreqNet test set", "image", + ["/hddstore/ssalehi/SynthScope/freqnet"], ["test"], + "FreqNet evaluation set (ForenSynths-style real/fake)."), + DatasetDef("universialfakedetection", "UniversalFakeDetect (Ojha)", "image", + ["/hddstore/ssalehi/SynthScope/universialfakedetection"], ["test"], + "Ojha et al. CVPR'23 cross-generator test set."), + # Cross-generator benchmarks added for the full reproduction suite. + DatasetDef("genimage_full", "GenImage (8 generators)", "image", + ["/hddstore/ssalehi/SynthScope/GenImage_full"], ["test"], + "Full 8-generator GenImage test (Midjourney, SD-v1.4/1.5, ADM, GLIDE, Wukong, VQDM, BigGAN)."), + DatasetDef("synthbuster", "Synthbuster (9 diffusion generators)", "image", + ["/hddstore/ssalehi/SynthScope/Synthbuster"], ["test"], + "Synthbuster (Zenodo 10066460) — DALL-E 2/3, Firefly, GLIDE, Midjourney-v5, SD 1.3/1.4/2/XL; " + "used by SPAI. Proxy reals (GenImage nature) since Synthbuster ships fakes only."), + DatasetDef("diffusionforensics", "DiffusionForensics / DIRE", "image", + ["/hddstore/ssalehi/SynthScope/DiffusionForensics"], ["test"], + "8 diffusion subsets (ADM, DDPM, IDDPM, LDM, PNDM, SDv1, SDv2, VQ-Diffusion); NPR Table 3."), + DatasetDef("diffusion1kstep", "Diffusion-1kStep", "image", + ["/hddstore/ssalehi/SynthScope/Diffusion1kStep"], ["test"], + "DALLE, DDPM, guided-diffusion, improved-diffusion, midjourney; NPR Table 5."), + DatasetDef("aigcdetect", "AIGCDetectBenchmark (16 gen)", "image", + ["/hddstore/ssalehi/SynthScope/AIGCDetectBenchmark"], ["test"], + "16-generator GAN+diffusion benchmark; Co-Spy / AIDE cross-generator table."), + DatasetDef("commgen15", "CommGen15 (15 commercial gen)", "image", + ["/hddstore/ssalehi/SynthScope/CommGen15"], ["test"], + "15 commercial generators (Midjourney, Sora, Flux, Kling, ...); PGC Table 3."), + DatasetDef("videofeedback", "VideoFeedback (TIGER-Lab)", "video", + ["/hddstore/ssalehi/SynthScope/VideoFeedback"], ["test"], + "11 video generators vs DiDeMo+Panda70M real; STALL benchmark."), + DatasetDef("comgenvid", "ComGenVid (Sora/VEO3)", "video", + ["/hddstore/ssalehi/SynthScope/ComGenVid"], ["test"], + "Sora + VEO3 fakes vs MSVD real; STALL's new benchmark."), +] + +# Which dataset each detector was trained on (best-effort, from interface/scripts/train_model.sh +# and experiments/*). `official` flags checkpoints whose deployed weights are the authors' (the +# SynthScope from-scratch dataset is noted, but the served weights are the official ProGAN-trained ones). +MODEL_TRAIN_DATASET: dict[str, dict] = { + "clip": {"dataset": "cnndetection"}, + "cnn": {"dataset": "cnndetection"}, + "dinov2": {"dataset": "cnndetection"}, + "dinov3": {"dataset": "cnndetection"}, + "adapted_dino": {"dataset": "cnndetection"}, + "safe": {"dataset": "cnndetection"}, + "pgc": {"dataset": "cospy_subset"}, + "co_spy": {"dataset": "cospy_subset"}, + "vib_net": {"dataset": "genimage"}, + "aide": {"dataset": "genimage", "official": True}, + "npr": {"dataset": "genimage", "official": True}, + "spai": {"dataset": "spai_ldm"}, + "fat_former": {"dataset": "cnndetection", "official": True}, + "restrav": {"dataset": "restrav_data"}, + "d3": {"dataset": None, "training_free": True}, + "stall": {"dataset": None, "training_free": True}, +} + + +# Node-local scratch differs per node (cn023 -> /hddstore, cees -> /ssdstore, cn030/31 -> /nvmestore). +# Try each configured root, then the same tail remapped onto the other scratch mounts, so the explorer +# works wherever the data was staged. Override the prefix set with SYNTHSCOPE_DATA_PREFIXES (colon-separated). +_SCRATCH_PREFIXES = tuple(p for p in ( + os.environ.get("SYNTHSCOPE_DATA_PREFIXES") or "/hddstore:/ssdstore:/nvmestore").split(":") if p) + + +def _root_candidates(r: str) -> list[str]: + cands = [r] + # SYNTHSCOPE_DATA_DIR: one folder holding datasets by their usual folder name (e.g. /CommGen15), as on a Space + if os.environ.get("SYNTHSCOPE_DATA_DIR"): + cands.insert(0, str(Path(os.environ["SYNTHSCOPE_DATA_DIR"]).expanduser() / Path(r).name)) + for src in _SCRATCH_PREFIXES: + if r.startswith(src + "/"): + tail = r[len(src):] + cands += [pre + tail for pre in _SCRATCH_PREFIXES if pre != src] + break + return cands + + +def _resolve_root(d: DatasetDef) -> Path | None: + for r in d.roots: + for c in _root_candidates(r): + p = Path(c) + if p.is_dir(): + return p + return None + + +def _exts_for(modality: str) -> set[str]: + return VIDEO_EXTS if modality == "video" else IMAGE_EXTS + + +def _split_dir(root: Path, split: str) -> Path: + return root if split in (".", "", None) else root / split + + +def label_of(rel: Path) -> str | None: + """Infer 'real'/'fake' from a relative path's directory tokens (None if unknown).""" + parts = [p.lower() for p in rel.parts] + for p in parts: + if p in FAKE_TOKENS: + return "fake" + for p in parts: + if p in REAL_TOKENS: + return "real" + return None + + +def available_splits(d: DatasetDef, root: Path) -> list[str]: + out = [] + for s in d.splits: + if _split_dir(root, s).is_dir(): + out.append(s) + return out + + +def list_datasets() -> list[dict]: + """All registry datasets whose data is present on disk, with resolved path + splits.""" + out = [] + for d in DATASET_DEFS: + root = _resolve_root(d) + if root is None: + continue + out.append({ + "key": d.key, + "name": d.name, + "modality": d.modality, + "note": d.note, + "path": str(root), + "splits": available_splits(d, root), + }) + return out + + +def get_def(key: str) -> DatasetDef: + for d in DATASET_DEFS: + if d.key == key: + return d + raise KeyError(f"Unknown dataset '{key}'") + + +def _iter_files(base: Path, exts: set[str]): + scanned = 0 + for dirpath, _dirs, files in os.walk(base): + for fn in files: + if Path(fn).suffix.lower() in exts: + yield Path(dirpath) / fn + scanned += 1 + if scanned >= SCAN_CAP: + return + + +def _pools(base: Path, exts: set[str]) -> dict[str, list[Path]]: + """Reservoir-sample up to POOL_PER_LABEL paths per label across the whole tree.""" + pools: dict[str, list[Path]] = {"real": [], "fake": []} + seen: dict[str, int] = {"real": 0, "fake": 0} + rng = random.Random(1234) + for fp in _iter_files(base, exts): + lab = label_of(fp.relative_to(base)) + if lab is None: + continue + seen[lab] += 1 + pool = pools[lab] + if len(pool) < POOL_PER_LABEL: + pool.append(fp) + else: # reservoir replacement + j = rng.randint(0, seen[lab] - 1) + if j < POOL_PER_LABEL: + pool[j] = fp + return pools + + +def sample_batch(key: str, split: str, n: int, seed: int) -> list[dict]: + """Return ``n`` random items (balanced real/fake when possible) from ``key``/``split``.""" + d = get_def(key) + root = _resolve_root(d) + if root is None: + raise FileNotFoundError(f"Dataset '{key}' not present on disk") + base = _split_dir(root, split) + if not base.is_dir(): + raise FileNotFoundError(f"Split '{split}' not found for '{key}'") + pools = _pools(base, _exts_for(d.modality)) + rng = random.Random(seed) + half = max(1, n // 2) + items: list[Path] = [] + for lab in ("real", "fake"): + p = pools[lab] + rng.shuffle(p) + items += [(fp, lab) for fp in p[:half]] + # top up if one class is short + if len(items) < n: + extra = [(fp, lab) for lab in ("real", "fake") for fp in pools[lab][half:]] + rng.shuffle(extra) + items += extra[: n - len(items)] + rng.shuffle(items) + items = items[:n] + return [ + {"rel": str(fp.relative_to(root)), "label": lab, + "name": fp.name, "modality": d.modality} + for fp, lab in items + ] + + +def resolve_media(key: str, rel: str) -> Path: + """Resolve ``rel`` under the dataset root, guarding against ``..`` traversal. + + Some datasets (e.g. ``restrav_data``) populate their split dirs with *symlinks* into other + SynthScope data folders, so we must NOT ``.resolve()`` and then reject by the symlink's target + (that would 403 every restrav_data thumbnail). Instead we forbid absolute paths and ``..`` + components in the user-supplied ``rel`` — which keeps the addressed path lexically under the + dataset root — then follow the symlink (``is_file``) to serve the actual file. + """ + d = get_def(key) + root = _resolve_root(d) + if root is None: + raise FileNotFoundError(f"Dataset '{key}' not present on disk") + rel_path = Path(rel) + if rel_path.is_absolute() or ".." in rel_path.parts: + raise PermissionError("Path escapes dataset root") + target = root / rel_path + if not target.is_file(): # follows the symlink to the real video/image + raise FileNotFoundError(str(target)) + return target diff --git a/interface/backend/engine.py b/interface/backend/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..dc32cde712517ddb30cef749977c0595d743b6dc --- /dev/null +++ b/interface/backend/engine.py @@ -0,0 +1,409 @@ +"""Inference engine: build SynthScope models from config, load checkpoints, and predict. + +Models are built lazily on first use (backbones are large and slow to download/instantiate) +and kept in a small LRU so a shared GPU is not exhausted. All forward passes happen under +``torch.no_grad()`` with the model in fp32/eval mode. +""" +import base64 +import gc +import io +import threading +import time +from collections import OrderedDict + +import numpy as np +import torch +import torchvision.transforms as T +from PIL import Image + +from . import config +from .registry import ModelSpec, get_spec, input_note +from . import media + + +def _png_b64(img: Image.Image) -> str: + buf = io.BytesIO() + img.save(buf, format="PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii") + + +def _saliency_rgb(sal: torch.Tensor): + """2-D saliency -> (jet RGB uint8 HxWx3, normalized intensity HxW in [0,1]). + + Robust-normalizes by the 1st/99th percentile so a few hot pixels don't wash it out. + """ + s = sal.detach().float().cpu().numpy() + lo, hi = np.percentile(s, 1.0), np.percentile(s, 99.0) + s = np.clip((s - lo) / (hi - lo + 1e-8), 0.0, 1.0) + r = np.clip(1.5 - np.abs(4 * s - 3), 0, 1) # jet colormap + g = np.clip(1.5 - np.abs(4 * s - 2), 0, 1) + b = np.clip(1.5 - np.abs(4 * s - 1), 0, 1) + return (np.stack([r, g, b], axis=-1) * 255).astype(np.uint8), s + + +def _saliency_to_png_b64(sal: torch.Tensor, out_size: int = 288) -> str: + """Render a saliency tensor as a base64 'jet' heatmap PNG.""" + rgb, _ = _saliency_rgb(sal) + return _png_b64(Image.fromarray(rgb, "RGB").resize((out_size, out_size), Image.BILINEAR)) + + +def _overlay_to_png_b64(sal: torch.Tensor, base: Image.Image, out_size: int = 288, + alpha: float = 0.6) -> str: + """Blend the jet heatmap over the base image; per-pixel opacity = saliency intensity so cool + regions show the image and hot regions show the artifact heat.""" + rgb, s = _saliency_rgb(sal) + heat = Image.fromarray(rgb, "RGB").resize((out_size, out_size), Image.BILINEAR) + base = base.convert("RGB").resize((out_size, out_size), Image.BILINEAR) + mask = Image.fromarray((np.clip(s, 0, 1) * alpha * 255).astype(np.uint8), "L").resize( + (out_size, out_size), Image.BILINEAR) + return _png_b64(Image.composite(heat, base, mask)) + + +def _geometric_base(spec, pil: Image.Image) -> Image.Image: + """The model's input region in RGB (apply only the Resize/CenterCrop steps of the eval + transform, skipping ToTensor/Normalize) so the overlay aligns with the saliency. Custom + transforms (AIDE/SPAI native-res) fall back to the native image.""" + import torchvision.transforms as T + base = pil.convert("RGB") + for t in getattr(spec.transform, "transforms", []) or []: + if isinstance(t, (T.Resize, T.CenterCrop)): + base = t(base) + return base + + +# --------------------------------------------------------------------------- +# Model construction (mirrors trainer.SynthScopeModelTrainer._build_model dispatch, with lazy +# imports so unused heavy dependencies are never loaded). +# --------------------------------------------------------------------------- +def _build_model(cfg): + model_type = cfg.ModelConfig.get("model_type", "dino").lower() + if model_type == "safe": + from models.image.safe.safe import SynthScopeSAFEModel + return SynthScopeSAFEModel(cfg) + if model_type == "fat_former": + from models.image.fat_former.fat_former import SynthScopeFatFormerModel + return SynthScopeFatFormerModel(cfg) + if model_type == "co_spy": + from models.image.co_spy.co_spy import SynthScopeCoSpyModel + return SynthScopeCoSpyModel(cfg) + if model_type == "aide": + from models.image.aide.aide import SynthScopeAIDEModel + return SynthScopeAIDEModel(cfg) + if model_type == "spai": + from models.image.spai.spai import SPAI + return SPAI(cfg) + if model_type == "vib_net": + from models.image.vib_net.vib_net import SynthScopeVIBNetModel + return SynthScopeVIBNetModel(cfg) + if model_type == "npr": + from models.image.npr.npr import NPR + return NPR(cfg) + if model_type == "pgc": + from models.image.pgc.pgc import SynthScopePGCModel + return SynthScopePGCModel(cfg) + if model_type == "cure": + from models.image.cure.cure import SynthScopeCuReModel + return SynthScopeCuReModel(cfg) + if model_type == "stall": + from models.video.stall.stall import SynthScopeSTALLModel + return SynthScopeSTALLModel(cfg) + if model_type == "d3": + from models.video.d3.d3 import SynthScopeD3Model + return SynthScopeD3Model(cfg) + if model_type == "restrav": + from models.video.restrav.restrav import SynthScopeReStraVModel + return SynthScopeReStraVModel(cfg) + if model_type == "cnn": + from models.image.cnn_detection.cnn_detection import SynthScopeCNNModel + return SynthScopeCNNModel(cfg) + if model_type == "adapted_dino": + from models.image.adapted_dino.adapted_dino import SynthScopeAdaptedDINOModel + return SynthScopeAdaptedDINOModel(cfg) + if model_type == "dinov2": + from models.image.dino_detection.dino_detection import SynthScopeDINOModel + return SynthScopeDINOModel(cfg) + if model_type == "dinov3": + from models.image.dino_detection.dino_detection import SynthScopeDINOv3Model + return SynthScopeDINOv3Model(cfg) + if model_type == "clip" or cfg.ModelConfig.backbone_name.startswith("CLIP:"): + from models.image.clip_detection.clip_detection import SynthScopeCLIPModel + return SynthScopeCLIPModel(cfg) + raise ValueError(f"Invalid model_type: {model_type}") + + +def _load_checkpoint(model, spec: ModelSpec): + """Load weights for ``spec`` into ``model`` (already constructed). Returns a status string.""" + path = config.CHECKPOINTS_DIR / spec.checkpoint + if not path.exists(): + raise FileNotFoundError( + f"Checkpoint not found: {path}. Train or place the checkpoint, then retry." + ) + raw = torch.load(str(path), map_location="cpu", weights_only=False) + sd = raw["model"] if isinstance(raw, dict) and "model" in raw else raw + if spec.remap is not None: + sd = spec.remap(sd) + missing, unexpected = model.load_state_dict(sd, strict=False) + # Buffers that are legitimately reconstructed in __init__ / not in official dumps. + benign = {"clip_mean", "clip_std", "imnet_mean", "imnet_std"} + real_missing = [ + k for k in missing + if not k.startswith("criterion") and not k.startswith("dwt_filter.") + and k.split(".")[-1] not in benign + ] + if spec.ckpt_kind.startswith("official"): + # Official-weights loads are verified to be exact; flag any genuine mismatch loudly. + if real_missing or unexpected: + raise RuntimeError( + f"{spec.key}: checkpoint mismatch missing={real_missing[:8]} " + f"unexpected={unexpected[:8]}" + ) + return ( + f"loaded {len(sd)} tensors from {path.name} " + f"(missing={len(real_missing)}, unexpected={len(unexpected)})" + ) + + +class Engine: + """Lazily builds, caches, and runs the detectors.""" + + def __init__(self, device: str | None = None): + # ``device`` lets a caller (e.g. the ``synthscope`` Python API) pin a specific device; + # the web backend constructs it with no argument and auto-detects via config.device(). + self.device = device or config.device() + self._cache: "OrderedDict[str, torch.nn.Module]" = OrderedDict() + self._status: dict[str, str] = {} + self._lock = threading.Lock() + + # -- availability ------------------------------------------------------- + def checkpoint_path(self, spec: ModelSpec): + if spec.checkpoint is None: + return None + return config.CHECKPOINTS_DIR / spec.checkpoint + + def is_available(self, spec: ModelSpec) -> bool: + if spec.key == "stall": # training-free, but needs the gated DINOv3 ViT-L/16 weights + import os + from .registry import DINOV3_WEIGHTS + return os.path.exists(os.environ.get("DINO_V3_WEIGHTS") or DINOV3_WEIGHTS) + if spec.ckpt_kind == "none": # training-free (D3) + return True + p = self.checkpoint_path(spec) + return bool(p and p.exists()) + + # -- model lifecycle ---------------------------------------------------- + def release_gpu(self) -> None: + """Drop every resident detector (they reload lazily on the next request) so a heavy local image generator, + which runs in its own process and wants most of the GPU, does not collide with them.""" + with self._lock: + self._cache.clear() + gc.collect() + if self.device == "cuda": + torch.cuda.empty_cache() + + def _get_model(self, spec: ModelSpec) -> torch.nn.Module: + with self._lock: + if spec.key in self._cache: + self._cache.move_to_end(spec.key) + return self._cache[spec.key] + + # Evict LRU entries if at capacity. + while len(self._cache) >= config.MAX_RESIDENT_MODELS: + old_key, old_model = self._cache.popitem(last=False) + del old_model + if self.device == "cuda": + torch.cuda.empty_cache() + + from config_utils import compose_config + + cfg = compose_config(spec.overrides) + model = _build_model(cfg) + if spec.ckpt_kind != "none": + self._status[spec.key] = _load_checkpoint(model, spec) + else: + self._status[spec.key] = "training-free (no checkpoint)" + model = model.to(self.device).float().eval() + self._cache[spec.key] = model + return model + + # -- prediction helpers ------------------------------------------------- + @torch.no_grad() + def _fake_prob(self, model, spec: ModelSpec, x: torch.Tensor) -> torch.Tensor: + """Return P(fake) per sample for an image batch ``x`` [B,3,H,W].""" + kind = spec.predict_kind + if kind == "ce": + return model(x).softmax(dim=1)[:, 1] + if kind == "bce": + return torch.sigmoid(model(x).reshape(x.shape[0])) + if kind == "bce_npr": + logits = model.forward(model.NPR_transform(x)).reshape(x.shape[0]) + return torch.sigmoid(logits) + if kind == "vib": + out = model(x) + logit = out[1] if isinstance(out, (tuple, list)) else out + return torch.sigmoid(logit.reshape(x.shape[0])) + raise ValueError(f"Unexpected image predict_kind: {kind}") + + def _d3_calibration(self): + """Load (and cache) the D3 score->probability calibration if present.""" + if not hasattr(self, "_d3_cal"): + import json + # A file in the checkpoint folder overrides the one shipped with the model (fitted on GenVideo-Val). + p = config.CHECKPOINTS_DIR / "d3_calibration.json" + if not p.exists(): + p = config.REPO_ROOT / "models" / "video" / "d3" / "d3_calibration.json" + self._d3_cal = json.loads(p.read_text()) if p.exists() else None + return self._d3_cal + + @torch.no_grad() + def _video_fake_prob(self, model, spec: ModelSpec, clip: torch.Tensor) -> torch.Tensor: + """Return P(fake) for a single normalized clip tensor [T,3,H,W].""" + frames = clip.unsqueeze(0).to(self.device) # [1,T,3,H,W] + if spec.key == "d3": + cal = self._d3_calibration() + if cal is not None: + s = model.score(frames) # [1] raw volatility score + z = (s - cal["threshold"]) / cal["scale"] + if not cal["higher_is_fake"]: + z = -z + return torch.sigmoid(z) + out = model(frames) # [1,2] + if spec.key == "stall": + # STALL returns [real_prob, fake_prob] (already calibrated percentiles); use directly. + return out[:, 1] + return out.softmax(dim=1)[:, 1] + + # -- public API --------------------------------------------------------- + @torch.no_grad() + def _interpret_map(self, model, spec: ModelSpec, x: torch.Tensor, pil=None): + """Return ``{"map": heatmap_b64, "overlay": overlay_b64}`` for the detector's artifact map, + or None. ``map`` is the bare jet heatmap; ``overlay`` blends it on the model-input image. + + Best-effort + isolated: any failure returns None and never affects the prediction. + ``x`` is the already-transformed [1,3,H,W] input; ``pil`` is the native image (some maps, + e.g. AIDE's native-res DCT energy, are computed from the original resolution). + """ + kind = getattr(spec, "interpret", "") or "" + if not kind: + return None + try: + if kind == "safe_hf": # SAFE: DWT HH high-frequency band magnitude + hh = model._preprocess_high_frequency(x) # [1,3,H,W] + sal = hh.abs().mean(dim=1)[0] # [H,W] + elif kind == "npr_residual": # NPR: neighboring-pixel residual magnitude + res = model.NPR_transform(x) # [1,3,H,W] + sal = res.abs().mean(dim=1)[0] # [H,W] + elif kind == "pgc_peak": # PGC: peak-guided residual fake-evidence map + sal = model.peak_residual_map(x) # [Hr,Wr] + elif kind == "aide_dct": # AIDE: native-res DCT per-window energy (patch selection) + if pil is None: + return None + sal = spec.transform.dct_energy_map(pil) # [nH,Wn] + elif kind == "spai_patch": # SPAI: per-patch spectral fake-evidence over the 224-grid + sal = model.patch_fake_map(x) # [nH,nW] or None (five-crop/large grid) + if sal is None: + return None + else: + return None + base = _geometric_base(spec, pil) if pil is not None else None + out = {"map": _saliency_to_png_b64(sal)} + if base is not None: + out["overlay"] = _overlay_to_png_b64(sal, base) + return out + except Exception: # noqa: BLE001 — interpretability is optional, never break predict + return None + + @staticmethod + def _model_input_data_url(spec: ModelSpec, x: torch.Tensor, max_side: int = 1280) -> str: + """The image exactly as the network receives it (after this detector's preprocessing), as a PNG data URL. + Normalization is undone for display; multi-view inputs (AIDE) are laid out side by side.""" + t = x[0].detach().float().cpu() + if t.dim() == 4: + t = torch.cat(list(t), dim=2) + norm = next((s for s in getattr(spec.transform, "transforms", []) if isinstance(s, T.Normalize)), None) + if norm is not None: + t = t * torch.tensor(norm.std)[:, None, None] + torch.tensor(norm.mean)[:, None, None] + elif spec.key == "cure": # CuRe maps pixels to (x - 127.5) / 127.5 + t = t * 0.5 + 0.5 + if t.min() < 0 or t.max() > 1: # unknown scaling: stretch for display only + t = (t - t.min()) / (t.max() - t.min() + 1e-8) + img = Image.fromarray((t.clamp(0, 1) * 255).round().byte().permute(1, 2, 0).numpy()) + if max(img.size) > max_side: + k = max_side / max(img.size) + img = img.resize((max(1, int(img.width * k)), max(1, int(img.height * k))), Image.LANCZOS) + buf = io.BytesIO(); img.save(buf, "PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + + def predict_image(self, spec: ModelSpec, pil: Image.Image) -> dict: + model = self._get_model(spec) + x = spec.transform(pil).unsqueeze(0).to(self.device) + t0 = time.time() + p = float(self._fake_prob(model, spec, x).item()) + res = self._result(spec, p, time.time() - t0) + res["orig_size"] = [pil.width, pil.height] + res["input_size"] = [int(x.shape[-1]), int(x.shape[-2])] # what the network actually received + res["input_note"] = input_note(spec) + try: + res["input_image"] = self._model_input_data_url(spec, x) + except Exception: # display-only: never fail a prediction over it + pass + amap = self._interpret_map(model, spec, x, pil) + if amap is not None: + res["interpret_map"] = amap["map"] + if amap.get("overlay"): + res["interpret_overlay"] = amap["overlay"] + res["interpret_kind"] = spec.interpret + return res + + def predict_video_frames(self, spec: ModelSpec, frames: list[Image.Image]) -> dict: + """Run an *image* detector on sampled video frames and aggregate (mean P(fake)).""" + model = self._get_model(spec) + batch = torch.stack([spec.transform(f) for f in frames]).to(self.device) + t0 = time.time() + per_frame = self._fake_prob(model, spec, batch).detach().float().cpu().tolist() + elapsed = time.time() - t0 + mean_p = sum(per_frame) / len(per_frame) + res = self._result(spec, mean_p, elapsed) + res["per_frame"] = [round(v, 6) for v in per_frame] + res["aggregation"] = "mean over sampled frames" + return res + + def predict_video_clip(self, spec: ModelSpec, path) -> dict: + """Run a native *video* detector on a decoded clip.""" + model = self._get_model(spec) + clip = media.video_clip_tensor( + path, num_frames=spec.num_frames, fps=spec.fps, frame_size=spec.frame_size, + window_center=spec.window_center, interp=spec.resize_interp, + edge_crop=getattr(spec, "edge_crop", 0.0), + ) + mean, std = spec.video_norm + norm = torch.tensor(mean).view(1, 3, 1, 1) + sd = torch.tensor(std).view(1, 3, 1, 1) + clip = (clip - norm) / sd + t0 = time.time() + p = float(self._video_fake_prob(model, spec, clip).item()) + res = self._result(spec, p, time.time() - t0) + res["num_frames"] = spec.num_frames + return res + + def _result(self, spec: ModelSpec, fake_prob: float, elapsed: float) -> dict: + fake_prob = max(0.0, min(1.0, fake_prob)) + return { + "model": spec.key, + "model_name": spec.name, + "fake_probability": round(fake_prob, 6), + "real_probability": round(1.0 - fake_prob, 6), + "verdict": "fake" if fake_prob >= 0.5 else "real", + "confidence": round(max(fake_prob, 1.0 - fake_prob), 6), + "elapsed_ms": round(elapsed * 1000, 1), + "checkpoint_status": self._status.get(spec.key, ""), + } + + +_ENGINE: Engine | None = None + + +def get_engine() -> Engine: + global _ENGINE + if _ENGINE is None: + _ENGINE = Engine() + return _ENGINE diff --git a/interface/backend/generators.py b/interface/backend/generators.py new file mode 100644 index 0000000000000000000000000000000000000000..9f5ce5de9e9e0e600be178a469f77a1f73d692a4 --- /dev/null +++ b/interface/backend/generators.py @@ -0,0 +1,362 @@ +"""SOTA image-generation backends for the live detector-comparison demo. + +Each generator takes a source image + a prompt and returns PNG bytes of a freshly generated image. Two modes: + * "recreate" (default): a captioner describes the source photo and the generator creates a NEW image from that + description alone (image_png=None), so every pixel is synthesized. This is how detection benchmarks build fakes. + * "edit": the generator edits the source photo itself (image-conditioned). The result keeps the original's layout. +The result is then scored by the SynthScope detectors, which is then scored by the SynthScope detectors so a +user can see how today's detectors hold up against today's generators. API keys come from env vars; +if a key is absent the generator is reported unavailable (the UI greys it out). +""" +from __future__ import annotations +import base64, io, os, re, subprocess, sys, tempfile, threading +import _synthscope_env # noqa: F401 (maps legacy AGID_* variables to SYNTHSCOPE_*) +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional + +import requests +from PIL import Image + +DEFAULT_PROMPT = "make another image conceptually similar to this" + + +@dataclass +class Generator: + key: str + name: str + env_key: str # API-key env var (hosted generators); HF token for the gated download (local) + uses_image: bool # True = image-conditioned (edit); False = text-to-image only + call: Callable[..., bytes] # call(image_png | None, prompt, size=None); image_png=None = text-only + note: str = "" + local: bool = False # True = open-weight, runs locally (no paid API key) + avail: Optional[Callable[[], bool]] = None # custom availability check (overrides the defaults) + heavy: bool = False # True = needs (most of) the GPU: the web engine releases its detectors first + edit_only: bool = False # True = an editing model that cannot create an image from text alone + + @property + def modes(self) -> list[str]: + if not self.uses_image: + return ["recreate"] + return ["edit"] if self.edit_only else ["recreate", "edit"] + + def needs_gpu(self) -> bool: + """True when this local model cannot run here because the server has no GPU (on a CPU, even the smallest one, + FLUX.2 [klein] 4B, takes over 30 minutes per image).""" + return self.local and not _has_gpu() + + def available(self) -> bool: + if self.needs_gpu(): + return False + if self.avail is not None: + return self.avail() + if self.local: # local = available if the inference pipeline is importable + try: + from diffusers import Ideogram4Pipeline # noqa: F401 + return True + except Exception: + return False + return bool(os.environ.get(self.env_key)) + + +def _has_gpu() -> bool: + try: + import torch + return torch.cuda.is_available() + except Exception: + return False + + +def _png_bytes(img: Image.Image) -> bytes: + buf = io.BytesIO(); img.convert("RGB").save(buf, "PNG"); return buf.getvalue() + + +# --- Image captioning (BLIP) for text-to-image generators ---------------------------------------- +# Text-to-image generators (Grok, Ideogram) can't see the source image, so the edit-style default +# prompt ("make another image conceptually similar to this") is meaningless to them and tends to +# produce garbage or trip the model's safety filter. We caption the source image and feed THAT, so +# the result is genuinely "conceptually similar" and the prompt is a normal scene description. +_BLIP = None + + +def _caption(image_png: bytes) -> Optional[str]: + """Best-effort caption of the source image. Returns None on any failure (callers fall back).""" + global _BLIP + try: + import torch + if _BLIP is None: + from transformers import BlipForConditionalGeneration, BlipProcessor + name = "Salesforce/blip-image-captioning-base" + _BLIP = (BlipProcessor.from_pretrained(name), + BlipForConditionalGeneration.from_pretrained(name).eval()) + proc, model = _BLIP + img = Image.open(io.BytesIO(image_png)).convert("RGB") + with torch.no_grad(): + out = model.generate(**proc(img, return_tensors="pt"), max_new_tokens=30) + cap = proc.decode(out[0], skip_special_tokens=True).strip() + return cap or None + except Exception: + return None + + +STYLE_SUFFIX = ", photorealistic photograph, natural lighting, high detail" + + +def describe_prompt(image_png: bytes) -> dict: + """Caption the source photo and turn it into a text-to-image prompt (shown to the user, who can edit it).""" + cap = _caption(image_png) + return {"caption": cap or "", "prompt": (cap + STYLE_SUFFIX) if cap else "a photorealistic natural scene, high detail"} + + +def effective_prompt(gen_key: str, image_png: bytes, prompt: str) -> str: + """The prompt actually sent to the generator. For text-to-image generators left on the default + edit instruction, derive a scene description from the source image (caption) instead.""" + g = GENERATORS.get(gen_key) + if g and not g.uses_image and prompt.strip() == DEFAULT_PROMPT: + cap = _caption(image_png) + return f"{cap}, photorealistic, high detail" if cap else "a photorealistic natural scene, high detail" + return prompt + + +# --- Google Gemini 2.5 Flash Image ("Nano Banana") — image-conditioned --------------------------- +def _gemini(image_png: Optional[bytes], prompt: str, size=None) -> bytes: + key = os.environ["GEMINI_API_KEY"] + url = ("https://generativelanguage.googleapis.com/v1beta/models/" + "gemini-2.5-flash-image:generateContent?key=" + key) + parts = [{"text": prompt}] + if image_png is not None: + parts.append({"inline_data": {"mime_type": "image/png", "data": base64.b64encode(image_png).decode()}}) + body = {"contents": [{"parts": parts}]} + r = requests.post(url, json=body, timeout=120); r.raise_for_status() + for part in r.json()["candidates"][0]["content"]["parts"]: + if "inline_data" in part or "inlineData" in part: + data = (part.get("inline_data") or part.get("inlineData"))["data"] + return base64.b64decode(data) + raise RuntimeError("Gemini returned no image part") + + +# --- OpenAI gpt-image-1 — image edit -------------------------------------------------------------- +def _openai(image_png: Optional[bytes], prompt: str, size=None) -> bytes: + key = os.environ["OPENAI_API_KEY"] + if image_png is None: # text-only: plain image generation + r = requests.post("https://api.openai.com/v1/images/generations", + headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, + json={"model": "gpt-image-1", "prompt": prompt, "size": "1024x1024"}, timeout=180) + r.raise_for_status() + return base64.b64decode(r.json()["data"][0]["b64_json"]) + r = requests.post("https://api.openai.com/v1/images/edits", + headers={"Authorization": f"Bearer {key}"}, + data={"model": "gpt-image-1", "prompt": prompt, "size": "1024x1024"}, + files={"image": ("src.png", image_png, "image/png")}, timeout=180) + r.raise_for_status() + return base64.b64decode(r.json()["data"][0]["b64_json"]) + + +# --- xAI Grok (grok-2-image) — text-to-image (no image input; uses the prompt) -------------------- +def _grok(image_png: Optional[bytes], prompt: str, size=None) -> bytes: + key = os.environ["XAI_API_KEY"] + r = requests.post("https://api.x.ai/v1/images/generations", + headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, + json={"model": "grok-2-image", "prompt": prompt, "response_format": "b64_json"}, + timeout=180) + r.raise_for_status() + return base64.b64decode(r.json()["data"][0]["b64_json"]) + + +# --- Black Forest Labs FLUX.1.1 (via fal.ai) — image-to-image ------------------------------------- +def _flux(image_png: Optional[bytes], prompt: str, size=None) -> bytes: + key = os.environ["FAL_KEY"] + if image_png is None: # text-only: FLUX.1 [dev] text-to-image + r = requests.post("https://fal.run/fal-ai/flux/dev", + headers={"Authorization": f"Key {key}", "Content-Type": "application/json"}, + json={"prompt": prompt, "image_size": "square_hd"}, timeout=180) + r.raise_for_status() + return requests.get(r.json()["images"][0]["url"], timeout=120).content + # fal.ai accepts a data-URI image_url for image-to-image + data_uri = "data:image/png;base64," + base64.b64encode(image_png).decode() + r = requests.post("https://fal.run/fal-ai/flux/dev/image-to-image", + headers={"Authorization": f"Key {key}", "Content-Type": "application/json"}, + json={"prompt": prompt, "image_url": data_uri, "strength": 0.85}, timeout=180) + r.raise_for_status() + img_url = r.json()["images"][0]["url"] + return requests.get(img_url, timeout=120).content + + +# --- Ideogram 4.0 — OPEN-WEIGHT, runs LOCALLY (no paid API key) ----------------------------------- +# Ideogram 4.0 (June 2026) is open-weight (9.3B DiT; Apache-2.0 inference code). The nf4 checkpoint +# (`ideogram-ai/ideogram-4-nf4-diffusers`) fits on a single 24 GB GPU. We load it via diffusers and +# run inference locally — no Ideogram API key. A free HF token (HF_TOKEN) is only needed ONCE to +# download the license-gated weights; after that it runs offline. Text-to-image (flow-matching DiT), +# so it generates from the prompt rather than conditioning on the source image. +_IDEOGRAM_PIPE = None + + +def _ideogram_local(image_png: Optional[bytes], prompt: str, size=None) -> bytes: + global _IDEOGRAM_PIPE + if _IDEOGRAM_PIPE is None: + import torch + from diffusers import Ideogram4Pipeline + pipe = Ideogram4Pipeline.from_pretrained( + "ideogram-ai/ideogram-4-nf4-diffusers", torch_dtype=torch.bfloat16, + token=os.environ.get("HF_TOKEN")) + _IDEOGRAM_PIPE = pipe.to("cuda" if __import__("torch").cuda.is_available() else "cpu") + out = _IDEOGRAM_PIPE(prompt=prompt, height=1024, width=1024).images[0] + return _png_bytes(out) + + +# --- Open-weight image-conditioned generators (FLUX.2 [klein], Qwen-Image-2.1), run in a SUBPROCESS ------ +# These are 20-60 GB repos that need a newer diffusers than the shared SynthScope env and would not fit on the +# web UI's 24 GB GPU next to the detectors, so each image is produced by interface/backend/local_gen.py in a +# separate process (model CPU-offload, memory fully released afterwards). Configure with: +# SYNTHSCOPE_GEN_PYTHONPATH dir holding diffusers main + huggingface_hub 1.x (prepended to the child PYTHONPATH) +# SYNTHSCOPE_GEN_HF_HOME node-local HF cache for the child (keeps tens of GB off the shared /ivi quota) +_LOCAL_GEN = Path(__file__).with_name("local_gen.py") +_LOCAL_LOCK = threading.Lock() # one heavy local generation at a time + + +def _local_gen_available() -> bool: + """The isolated generator environment is configured (the GPU requirement is checked per model).""" + extra = os.environ.get("SYNTHSCOPE_GEN_PYTHONPATH", "") + return bool(extra) and Path(extra).is_dir() + + +def _local_size(size) -> Optional[str]: + """'WxH' for the child: the source's aspect ratio at about one megapixel, sides in multiples of 16.""" + if not size: + return None + w, h = size + k = (1024 * 1024 / (w * h)) ** 0.5 + return f"{max(256, int(w * k) // 16 * 16)}x{max(256, int(h * k) // 16 * 16)}" + + +def _local_env() -> dict: + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join(p for p in (os.environ["SYNTHSCOPE_GEN_PYTHONPATH"], env.get("PYTHONPATH", "")) if p) + if os.environ.get("SYNTHSCOPE_GEN_HF_HOME"): + env["HF_HOME"] = os.environ["SYNTHSCOPE_GEN_HF_HOME"] + return env + + +def local_batch(model_key: str, items: list[dict], timeout: int = 6 * 3600) -> None: + """Generate many text-to-image items ({prompt, out, seed, size}) with ONE model load (used by the generator + benchmark). Outputs that already exist are skipped, so an interrupted run can be repeated.""" + import json + with _LOCAL_LOCK, tempfile.TemporaryDirectory() as tmp: + spec = Path(tmp) / "batch.json" + spec.write_text(json.dumps(items)) + r = subprocess.run([sys.executable, str(_LOCAL_GEN), "--model", model_key, "--batch", str(spec)], + env=_local_env(), capture_output=True, text=True, timeout=timeout) + if r.returncode != 0: + tail = (r.stderr or r.stdout or "").strip().splitlines()[-3:] + raise RuntimeError(f"{model_key} batch failed (exit {r.returncode}): " + " | ".join(tail)) + + +def _local_subprocess(model_key: str) -> Callable[..., bytes]: + def call(image_png: Optional[bytes], prompt: str, size=None) -> bytes: + env = _local_env() + with _LOCAL_LOCK, tempfile.TemporaryDirectory() as tmp: + src, dst = Path(tmp) / "in.png", Path(tmp) / "out.png" + cmd = [sys.executable, str(_LOCAL_GEN), "--model", model_key, "--prompt", prompt, "--out", str(dst)] + if image_png is None: + cmd.append("--text-only") + if _local_size(size): + cmd += ["--size", _local_size(size)] + else: + src.write_bytes(image_png) + cmd += ["--image", str(src)] + r = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=3600) + if r.returncode != 0 or not dst.exists(): + tail = (r.stderr or r.stdout or "").strip().splitlines()[-3:] + raise RuntimeError(f"{model_key} failed (exit {r.returncode}): " + " | ".join(tail)) + return dst.read_bytes() + call.model_key = model_key # lets the benchmark find the local model behind a generator + return call + + +GENERATORS = { + g.key: g for g in [ + Generator("nanobanana", "Nano Banana (Gemini 2.5 Flash Image)", "GEMINI_API_KEY", True, _gemini, + "Google's image-conditioned editor; respects the source image."), + Generator("gpt_image", "GPT Image (gpt-image-1)", "OPENAI_API_KEY", True, _openai, + "OpenAI image edit from the source."), + Generator("grok", "Grok (grok-2-image)", "XAI_API_KEY", False, _grok, + "xAI text-to-image; can't see the image, so it generates from an auto-caption of the source."), + Generator("flux", "FLUX.1 (img2img, via fal.ai)", "FAL_KEY", True, _flux, + "Black Forest Labs FLUX image-to-image."), + Generator("ideogram", "Ideogram 4.0 (open-weight, local)", "HF_TOKEN", False, _ideogram_local, + "Open-weight 9.3B DiT, nf4 on a 24GB GPU — runs locally via diffusers, NO paid API " + "key. HF_TOKEN only to download the license-gated weights once. Text-to-image: " + "generates from an auto-caption of the source (~4 min/image).", local=True), + Generator("flux2_klein", "FLUX.2 [klein] 4B (open-weight, local)", "SYNTHSCOPE_GEN_PYTHONPATH", True, + _local_subprocess("flux2_klein_4b"), + "Black Forest Labs' distilled FLUX.2 [klein] 4B (Apache-2.0), edits the source photo into a " + "new image locally on the GPU (4 steps); no API key.", local=True, + avail=_local_gen_available, heavy=True), + Generator("flux2_klein_9b", "FLUX.2 [klein] 9B (open-weight, local)", "SYNTHSCOPE_GEN_PYTHONPATH", True, + _local_subprocess("flux2_klein_9b"), + "Black Forest Labs' distilled FLUX.2 [klein] 9B (FLUX non-commercial license; gated - accept the " + "terms on the Hub), edits the source photo locally on the GPU (4 steps); no API key.", local=True, + avail=_local_gen_available, heavy=True), + Generator("flux1_kontext", "FLUX.1 Kontext [dev] (open-weight, local)", "SYNTHSCOPE_GEN_PYTHONPATH", True, + _local_subprocess("flux1_kontext_dev"), + "Black Forest Labs' 12B in-context image editor (FLUX non-commercial license; gated - accept the " + "terms on the Hub), edits the source photo locally on the GPU (28 steps); no API key. " + "Edit-only: it cannot create an image from text.", local=True, + avail=_local_gen_available, heavy=True, edit_only=True), + Generator("qwen_image_2_1", "Qwen-Image-2.1 (open-weight, local)", "SYNTHSCOPE_GEN_PYTHONPATH", True, + _local_subprocess("qwen_image_2_1"), + "Alibaba Qwen-Image-2.1 (7B unified generation + editing; Qwen Research License - research use " + "only), edits the source photo locally on the GPU; no API key.", local=True, + avail=_local_gen_available, heavy=True), + ] +} + + +def list_generators() -> list[dict]: + return [{"key": g.key, "name": g.name, "uses_image": g.uses_image, "modes": g.modes, + "available": g.available(), "needs_gpu": g.needs_gpu(), "local": g.local, + "env_key": g.env_key, "note": g.note} + for g in GENERATORS.values()] + + +# Text-to-image generators with a baked-in content policy (Ideogram) refuse photorealistic PEOPLE and +# return a flat gray "Image blocked by safety filter" placeholder. We pre-empt the obvious case (person +# in the caption) before wasting a full ~4-min generation, and detect the placeholder as a backstop. +_PERSON_RE = re.compile( + r"\b(person|people|man|men|woman|women|boy|girl|child|children|baby|lady|human|portrait|selfie|crowd|couple)\b", + re.I) +_STEER = "Pick a non-person image, or use an image-conditioned generator (Nano Banana / FLUX)." + + +def _looks_blocked(img: Image.Image) -> bool: + """A safety placeholder is near-uniform (text on a solid background): pixel std ~12 vs ~50 for a real image.""" + try: + import numpy as np + return float(np.asarray(img.convert("RGB"), dtype="float32").std()) < 22.0 + except Exception: + return False + + +def generate(gen_key: str, image_png: bytes, prompt: str = DEFAULT_PROMPT, mode: str = "recreate") -> bytes: + """mode "recreate": generate a new image from the text prompt alone; "edit": edit ``image_png`` itself.""" + if gen_key not in GENERATORS: + raise KeyError(f"unknown generator '{gen_key}'") + g = GENERATORS[gen_key] + if not g.available(): + raise RuntimeError(f"{g.name} unavailable: set {g.env_key}") + if mode not in ("recreate", "edit"): + raise RuntimeError(f"unknown mode '{mode}'") + if mode not in g.modes: + raise RuntimeError(f"{g.name} does not support the '{mode}' mode (it supports: {', '.join(g.modes)}).") + if not g.uses_image and _PERSON_RE.search(prompt or ""): + raise RuntimeError(f"{g.name} blocks photorealistic people, and this image reads as a photo of a " + f"person (“{prompt[:70]}”). {_STEER}") + size = Image.open(io.BytesIO(image_png)).size + out = g.call(image_png if mode == "edit" else None, prompt, size) + # normalize to PNG, cap size + img = Image.open(io.BytesIO(out)).convert("RGB") + if _looks_blocked(img): + raise RuntimeError(f"{g.name} returned a safety-filter placeholder — it blocked this content. {_STEER}") + if max(img.size) > 1024: + img.thumbnail((1024, 1024)) + return _png_bytes(img) diff --git a/interface/backend/local_gen.py b/interface/backend/local_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..18a0c83f93c5a8fb13f184c70ec694045a777462 --- /dev/null +++ b/interface/backend/local_gen.py @@ -0,0 +1,238 @@ +"""Isolated runner for heavy OPEN-WEIGHT image generators (FLUX.2 [klein] 4B/9B, FLUX.1 Kontext [dev], Qwen-Image-2.1). + +``generators.py`` launches this file as a SUBPROCESS, one image per call, so that + * these models can use a newer ``diffusers`` than the shared SynthScope env (set ``$SYNTHSCOPE_GEN_PYTHONPATH`` to a + directory holding diffusers main + huggingface_hub 1.x; see ``interface/README.md``), and + * every byte of GPU/CPU memory is released after the image, so the web UI's detectors keep their 24 GB. + + python interface/backend/local_gen.py --model qwen_image_2_1 --image in.png --prompt "..." --out out.png + +Both models are image-CONDITIONED (they edit the source photo into a new image), run in bf16 on a 24 GB GPU +(FLUX models: ``enable_model_cpu_offload``; Qwen-Image-2.1: staged load, see ``load_pipeline_staged``), and need +no paid API key. Licenses: FLUX.2 [klein] 4B is +Apache-2.0; Qwen-Image-2.1 is under the Qwen Research License (research use). +""" +import argparse +import gc +import json +import os +import sys + +import torch +import diffusers +import transformers +from PIL import Image + +MAX_AREA = 1024 * 1024 +MULTIPLE = 16 + +# key -> (HF repo, diffusers pipeline class, extra call kwargs) +MODELS = { + "flux2_klein_4b": ("black-forest-labs/FLUX.2-klein-4B", "Flux2KleinPipeline", + {"num_inference_steps": 4, "guidance_scale": 1.0}), + "qwen_image_2_1": ("Qwen/Qwen-Image-2.1", "QwenImage21Pipeline", + {"num_inference_steps": 40}), + # license-gated (accept the terms on the Hub with the account behind $HF_TOKEN); non-commercial licenses + "flux2_klein_9b": ("black-forest-labs/FLUX.2-klein-9B", "Flux2KleinPipeline", + {"num_inference_steps": 4, "guidance_scale": 1.0}), + "flux1_kontext_dev": ("black-forest-labs/FLUX.1-Kontext-dev", "FluxKontextPipeline", + {"num_inference_steps": 28, "guidance_scale": 2.5}), +} +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") +DEBUG = bool(os.environ.get("LOCAL_GEN_DEBUG")) + + +def mem(tag: str) -> None: + """GPU memory trace (set LOCAL_GEN_DEBUG=1).""" + if DEBUG: + print(f"[mem] {tag}: allocated {torch.cuda.memory_allocated() / 2**30:.1f} GiB, " + f"max {torch.cuda.max_memory_allocated() / 2**30:.1f} GiB", file=sys.stderr, flush=True) + + +def fit(img: Image.Image) -> Image.Image: + """Scale to <= MAX_AREA keeping aspect, with both sides a multiple of MULTIPLE.""" + w, h = img.size + s = min(1.0, (MAX_AREA / (w * h)) ** 0.5) + nw, nh = max(MULTIPLE, int(w * s) // MULTIPLE * MULTIPLE), max(MULTIPLE, int(h * s) // MULTIPLE * MULTIPLE) + return img.resize((nw, nh), Image.LANCZOS) + + +def load_pipeline_cpu_offload(repo: str, cls_name: str, token): + """Standard path: whole pipeline in host RAM, modules streamed to the GPU on demand (FLUX.2 [klein]). + Without a GPU the pipeline simply runs on the CPU in bfloat16 (slow: minutes per image).""" + pipe = getattr(diffusers, cls_name).from_pretrained(repo, torch_dtype=torch.bfloat16, token=token) + if DEVICE.type == "cuda": + pipe.enable_model_cpu_offload() + return pipe + + +def load_pipeline_staged(repo: str, cls_name: str, token): + """Low-host-RAM path for Qwen-Image-2.1 (33 GB repo, OOMs a 28 GB host with ``enable_model_cpu_offload``). + + The Qwen3-VL text encoder is loaded straight onto the GPU, used once to encode prompt + source image, then + dropped; only then is the 7B transformer loaded straight onto the GPU. Nothing large ever sits in host RAM. + The pipeline is created with ``transformer=None`` and the transformer is attached inside ``encode_prompt`` + (which ``__call__`` invokes before it first touches ``self.transformer``). + """ + pipeline_cls = getattr(diffusers, cls_name) + text_encoder = transformers.Qwen3VLForConditionalGeneration.from_pretrained( + repo, subfolder="text_encoder", torch_dtype=torch.bfloat16, device_map="cuda", token=token) + pipe = pipeline_cls.from_pretrained(repo, text_encoder=text_encoder, transformer=None, + torch_dtype=torch.bfloat16, token=token) + pipe.vae.to(DEVICE) + # The 1 MP VAE decode does not fit next to the 14 GB transformer on a 24 GB GPU. VAE tiling is NOT a fix: tiling the + # ENCODE of the conditioning image corrupts it (output degenerates into texture), and tiled decode leaves visible + # seams that a detector could key on. Instead the transformer (no longer needed once denoising is done) is moved + # to host RAM right before the single-pass decode. + vae_decode = pipe.vae.decode + + def decode_after_freeing_transformer(*args, **kwargs): + pipe.transformer.to("cpu") + gc.collect() + torch.cuda.empty_cache() + mem("transformer offloaded before decode") + return vae_decode(*args, **kwargs) + + pipe.vae.decode = decode_after_freeing_transformer + mem("text encoder + vae loaded") + pipeline_cls._execution_device = property(lambda self: DEVICE) # no offload hooks in this mode + held = {"text_encoder": text_encoder} # the only outside reference, so it can be dropped after encoding + del text_encoder + encode_prompt = pipe.encode_prompt + + def encode_then_load_transformer(*args, **kwargs): + out = encode_prompt(*args, **kwargs) + pipe.text_encoder = None + held.clear() + gc.collect() + torch.cuda.empty_cache() + mem("text encoder dropped") + pipe.register_modules(transformer=diffusers.QwenImage21Transformer2DModel.from_pretrained( + repo, subfolder="transformer", torch_dtype=torch.bfloat16, device_map="cuda", token=token)) + mem("transformer loaded") + return out + + pipe.encode_prompt = encode_then_load_transformer + return pipe + + +def load_pipeline_group_offload(repo: str, cls_name: str, token): + """FLUX.1 Kontext [dev]: the 12B transformer (24 GB in bf16) cannot sit whole on a 24 GB GPU, which is what + ``enable_model_cpu_offload`` needs. Keep the text encoders + VAE (~10 GB) on the GPU and stream the transformer + through it two blocks at a time.""" + pipe = getattr(diffusers, cls_name).from_pretrained(repo, torch_dtype=torch.bfloat16, token=token) + for name in ("text_encoder", "text_encoder_2", "vae"): + getattr(pipe, name).to(DEVICE) + pipe.transformer.enable_group_offload(onload_device=DEVICE, offload_device=torch.device("cpu"), + offload_type="block_level", num_blocks_per_group=2) + type(pipe)._execution_device = property(lambda self: DEVICE) # the transformer reports "cpu" between blocks + mem("kontext loaded") + return pipe + + +def run_batch_qwen(repo: str, cls_name: str, token, items: list, kw: dict, res: int) -> None: + """Text-to-image for MANY prompts with one model load (Qwen-Image-2.1 on a 24 GB GPU): encode every prompt first + with the text encoder on the GPU, drop it, load the transformer once, then denoise each prompt in turn + (the transformer is parked on the CPU during each VAE decode, as in ``load_pipeline_staged``).""" + pipeline_cls = getattr(diffusers, cls_name) + text_encoder = transformers.Qwen3VLForConditionalGeneration.from_pretrained( + repo, subfolder="text_encoder", torch_dtype=torch.bfloat16, device_map="cuda", token=token) + pipe = pipeline_cls.from_pretrained(repo, text_encoder=text_encoder, transformer=None, + torch_dtype=torch.bfloat16, token=token) + pipe.vae.to(DEVICE) + pipeline_cls._execution_device = property(lambda self: DEVICE) + embeds = [] + for it in items: + with torch.no_grad(): # without this the saved embeddings keep the autograd graph, and with it the 16 GB encoder + emb, mask, _ = pipe.encode_prompt(prompt=it["prompt"], device=DEVICE) + embeds.append((emb.detach().cpu(), None if mask is None else mask.detach().cpu())) + pipe.text_encoder = None + del text_encoder + gc.collect(); torch.cuda.empty_cache() + pipe.register_modules(transformer=diffusers.QwenImage21Transformer2DModel.from_pretrained( + repo, subfolder="transformer", torch_dtype=torch.bfloat16, device_map="cuda", token=token)) + vae_decode = pipe.vae.decode + + def decode_after_freeing_transformer(*args, **kwargs): + pipe.transformer.to("cpu"); gc.collect(); torch.cuda.empty_cache() + return vae_decode(*args, **kwargs) + + pipe.vae.decode = decode_after_freeing_transformer + for it, (emb, mask) in zip(items, embeds): + pipe.transformer.to(DEVICE) + out = pipe(prompt_embeds=emb.to(DEVICE), prompt_embeds_mask=None if mask is None else mask.to(DEVICE), + generator=torch.Generator("cpu").manual_seed(it.get("seed", 0)), output_resolution=res, **kw).images[0] + out.convert("RGB").save(it["out"], format="PNG") + print(f"saved {it['out']} {out.size}", flush=True) + + +def run_batch(model: str, items: list, res: int) -> None: + """Text-only generation for a list of {prompt, out, seed, size?} items with one model load. Existing outputs are + skipped, so an interrupted batch can simply be re-run.""" + repo, cls_name, kw = MODELS[model] + token = os.environ.get("HF_TOKEN") + todo = [it for it in items if not os.path.exists(it["out"])] + print(f"{len(items) - len(todo)} already done, {len(todo)} to generate with {model}", flush=True) + if not todo: + return + if model == "qwen_image_2_1": + run_batch_qwen(repo, cls_name, token, todo, kw, res) + return + pipe = load_pipeline_cpu_offload(repo, cls_name, token) + for it in todo: + w, h = (int(v) for v in (it.get("size") or "1024x1024").split("x")) + out = pipe(prompt=it["prompt"], generator=torch.Generator("cpu").manual_seed(it.get("seed", 0)), + height=h, width=w, **kw).images[0] + out.convert("RGB").save(it["out"], format="PNG") + print(f"saved {it['out']} {out.size}", flush=True) + + +LOADERS = {"qwen_image_2_1": load_pipeline_staged, "flux1_kontext_dev": load_pipeline_group_offload} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--model", required=True, choices=sorted(MODELS)) + ap.add_argument("--image", help="source photo (not needed with --text-only)") + ap.add_argument("--prompt") + ap.add_argument("--out") + ap.add_argument("--batch", help="JSON list of {prompt, out, seed, size}: text-to-image for all of them with one model load") + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--text-only", action="store_true", help="create the image from the prompt alone (no source photo)") + ap.add_argument("--size", help="WxH of the output in --text-only mode (default 1024x1024)") + ap.add_argument("--res", type=int, default=1024, help="output resolution (Qwen-Image-2.1 output_resolution)") + args = ap.parse_args() + if args.batch: + run_batch(args.model, json.load(open(args.batch)), args.res) + return 0 + if not args.prompt or not args.out: + ap.error("--prompt and --out are required (or use --batch)") + if not args.text_only and not args.image: + ap.error("--image is required unless --text-only is given") + + repo, cls_name, kw = MODELS[args.model] + if not hasattr(diffusers, cls_name): + print(f"diffusers {diffusers.__version__} has no {cls_name}; point $SYNTHSCOPE_GEN_PYTHONPATH at a newer diffusers", + file=sys.stderr) + return 3 + pipe = LOADERS.get(args.model, load_pipeline_cpu_offload)(repo, cls_name, os.environ.get("HF_TOKEN")) + + src = None if args.text_only else fit(Image.open(args.image).convert("RGB")) + call = dict(kw) + if args.model.startswith("flux"): + if src is not None: + call.update(height=src.height, width=src.width) + else: + w, h = (int(v) for v in (args.size or "1024x1024").split("x")) + call.update(height=h, width=w) + if args.model.startswith("qwen"): + call["output_resolution"] = args.res + cond = {} if src is None else {"image": src} + out = pipe(prompt=args.prompt, generator=torch.Generator("cpu").manual_seed(args.seed), **cond, **call).images[0] + out.convert("RGB").save(args.out, format="PNG") + print(f"saved {args.out} {out.size} via {cls_name} ({repo})", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/interface/backend/media.py b/interface/backend/media.py new file mode 100644 index 0000000000000000000000000000000000000000..12d56654de083e91ccdd8915817ac87b66eb47dd --- /dev/null +++ b/interface/backend/media.py @@ -0,0 +1,126 @@ +"""Image and video decoding helpers. + +Reuses the SynthScope repo's own decoders so the interface sees pixels exactly the way the +training/eval pipeline does: + * ``dataset._decode_rgb`` -> robust still-image decode (torchvision, PIL fallback) + * ``dataset.sample_video_frames`` -> evenly-sampled clip [T,3,size,size] float in [0,1] +""" +import io +from pathlib import Path + +import numpy as np +import torch +from PIL import Image, ImageFile + +# Some user uploads are truncated/progressive; let PIL load what it can. +ImageFile.LOAD_TRUNCATED_IMAGES = True + +# Faithful repo decoders. +from dataset import sample_video_frames # noqa: E402 +from models.image.cure.preprocess import SRC_FORMAT_KEY # noqa: E402 + +IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tif", ".tiff"} +VIDEO_EXTENSIONS = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".m4v"} + + +def media_kind(filename: str, content_type: str | None = None) -> str: + """Return ``"image"`` or ``"video"`` from the filename/extension (content_type as hint).""" + suffix = Path(filename).suffix.lower() + if suffix in VIDEO_EXTENSIONS: + return "video" + if suffix in IMAGE_EXTENSIONS: + return "image" + if content_type: + if content_type.startswith("video/"): + return "video" + if content_type.startswith("image/"): + return "image" + raise ValueError(f"Unsupported file type: {filename!r} (content_type={content_type})") + + +def decode_image(data: bytes) -> Image.Image: + """Decode raw bytes into an RGB PIL image.""" + source = Image.open(io.BytesIO(data)) + source_format = source.format + image = source.convert("RGB") + image.info[SRC_FORMAT_KEY] = source_format # lets format-aware transforms (CuRe) see PNG vs JPEG + return image + + +def sample_video_pil(path: str | Path, num_frames: int) -> list[Image.Image]: + """Sample ``num_frames`` frames spread across the whole video, at native resolution. + + Used when running an *image* detector on a video: each frame is treated exactly like an + uploaded still (the detector's own image transform handles resize/crop/normalize). + """ + import decord # local import: heavy, only needed for video + + decord.bridge.set_bridge("native") + reader = decord.VideoReader(str(path), ctx=decord.cpu(0)) + n_src = len(reader) + if n_src == 0: + raise ValueError("Could not decode any frames from the video.") + n = max(1, min(num_frames, n_src)) + indices = np.linspace(0, n_src - 1, num=n) + indices = np.clip(np.rint(indices).astype(np.int64), 0, n_src - 1).tolist() + frames = [] + for i in indices: + try: + arr = reader[i].asnumpy() # [H, W, 3] uint8 + frames.append(Image.fromarray(arr).convert("RGB")) + except Exception: + continue + if not frames: + raise ValueError("Could not decode any frames from the video.") + return frames + + +def thumbnail_bytes(path: str | Path, modality: str, size: int = 256) -> bytes: + """A small JPEG thumbnail for the dataset explorer grid. + + Image -> downscaled still; video -> its middle frame (decord), downscaled. Returns raw + JPEG bytes so the endpoint can stream them without writing temp files. + """ + if modality == "video": + import decord # heavy; only needed for video + decord.bridge.set_bridge("native") + reader = decord.VideoReader(str(path), ctx=decord.cpu(0)) + n = len(reader) + if n == 0: + raise ValueError("Could not decode any frames from the video.") + arr = reader[n // 2].asnumpy() # [H,W,3] uint8, middle frame + img = Image.fromarray(arr).convert("RGB") + else: + img = Image.open(path).convert("RGB") + img.thumbnail((size, size)) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=82) + return buf.getvalue() + + +def video_clip_tensor( + path: str | Path, + num_frames: int, + fps: float | None, + frame_size: int, + window_seconds: float = 2.0, + window_center: bool = False, + interp: str = "bilinear", + edge_crop: float = 0.0, +) -> torch.Tensor: + """Decode a clip for a *video* detector: [T, 3, frame_size, frame_size] float in [0, 1]. + + Delegates to the repo's ``sample_video_frames`` so inference sampling matches each detector's + training decode (e.g. ReStraV uses ``window_center=True, interp="bicubic"`` per the official repo; + D3 uses ``edge_crop=0.1`` to trim borders per the official D3 preprocessing). + """ + return sample_video_frames( + path, + num_frames=num_frames, + fps=fps, + window_seconds=window_seconds, + size=frame_size, + window_center=window_center, + interp=interp, + edge_crop=edge_crop, + ) diff --git a/interface/backend/registry.py b/interface/backend/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..d14a62f403d5411fd342c24bd4157e69a4e9121d --- /dev/null +++ b/interface/backend/registry.py @@ -0,0 +1,463 @@ +"""Model registry: the single source of truth for every detector the interface can run. + +For each detector we record: + * how to compose its SynthScope config (Hydra overrides, exactly like ``main.py``), + * its evaluation preprocessing (matched to the training/eval recipe or the verified + official-weights eval script under repro/), + * which checkpoint file backs it and how that checkpoint loads (strict SynthScope vs. an + official-weights key remap), and + * how to turn a forward pass into a fake-probability. + +Label convention (dataset.py:_get_label): index 0 = real, index 1 = fake. +""" +import functools +from dataclasses import dataclass, field +from typing import Callable, Optional + +import torchvision.transforms as T + +# --------------------------------------------------------------------------- +# Normalization constants (mirror transformations.py). +# --------------------------------------------------------------------------- +CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073) +CLIP_STD = (0.26862954, 0.26130258, 0.27577711) +IMAGENET_MEAN = (0.485, 0.456, 0.406) # == normalize_dinov2 +IMAGENET_STD = (0.229, 0.224, 0.225) +DINOV3_MEAN = (0.430, 0.411, 0.296) +DINOV3_STD = (0.213, 0.156, 0.143) + + +def _norm(mean, std): + return T.Normalize(mean=list(mean), std=list(std)) + + +def tf_resize_crop(resize: int, crop: int, mean=None, std=None) -> T.Compose: + """Resize shorter side -> ``resize``, center-crop ``crop``, ToTensor, optional Normalize.""" + steps = [T.Resize(resize), T.CenterCrop(crop), T.ToTensor()] + if mean is not None: + steps.append(_norm(mean, std)) + return T.Compose(steps) + + +def tf_crop(crop: int, mean=None, std=None) -> T.Compose: + """Center-crop ``crop`` with NO resize, ToTensor, optional Normalize. Official SAFE + preprocessing (scripts/eval.sh: --input_size 256 --transform_mode 'crop', no normalization) — + a resize would alter the high-frequency signal SAFE keys on.""" + steps = [T.CenterCrop(crop), T.ToTensor()] + if mean is not None: + steps.append(_norm(mean, std)) + return T.Compose(steps) + + +# AIDE's official native-resolution 5-view preprocessing (DCT patch selection on the native-size +# image). Importing this is light (no open_clip at module level); the heavy net is built lazily. +from models.image.aide.official_arch import AIDEFiveView # noqa: E402 +aide_five_view = AIDEFiveView() + +# CuRe's official eval preprocessing (PNG->JPEG96, 390 crop, 336 resize, JPEG70, mean/std 0.5), in cv2/PIL. +from models.image.cure.preprocess import CuReTransform # noqa: E402 + + +def _cap_long_side(im, max_side: int): + """Downscale a PIL image (keeping aspect, min side 224) only if its long side exceeds + ``max_side`` — otherwise return it untouched at native resolution.""" + w, h = im.size + if max(w, h) > max_side: + s = max_side / max(w, h) + im = im.resize((max(int(w * s), 224), max(int(h * s), 224))) + return im + + +def tf_native_cap(max_side: int = 1024, mean=None, std=None) -> T.Compose: + """Native/arbitrary-resolution preprocessing: keep the image at its NATIVE size, only + downscaling (keeping aspect, min side 224) when the long side exceeds ``max_side`` to bound + memory. ToTensor (raw [0,1]), optional Normalize. This is SPAI's official any-resolution + path — a fixed Resize/CenterCrop would destroy the native-res spectral statistics it learns. + NB: produces variable-size tensors, so a ``native_res=True`` spec must be scored ONE image at + a time (bs=1); the web ``predict_image`` already does this, and bench_eval forces bs=1.""" + steps = [T.Lambda(functools.partial(_cap_long_side, max_side=max_side)), T.ToTensor()] + if mean is not None: + steps.append(_norm(mean, std)) + return T.Compose(steps) + + +def tf_square_resize(resize: int, crop: Optional[int], mean=None, std=None) -> T.Compose: + """Square Resize((resize,resize)), optional center-crop, ToTensor, optional Normalize. + + Matches the official FatFormer/NPR/AIDE eval preprocessing (a square resize, not a + shorter-side resize).""" + steps = [T.Resize((resize, resize))] + if crop is not None: + steps.append(T.CenterCrop(crop)) + steps.append(T.ToTensor()) + if mean is not None: + steps.append(_norm(mean, std)) + return T.Compose(steps) + + +# --------------------------------------------------------------------------- +# Official-weights -> SynthScope state_dict remaps (copied verbatim from the verified repro +# eval scripts in /hddstore/ssalehi/SynthScope/repro/{fatformer,aide,npr}_eval.py). +# --------------------------------------------------------------------------- +def remap_fatformer(sd: dict) -> dict: + drop_pref = ("image_encoder.",) + drop_exact = { + "language_guided_alignment.token_prefix", + "language_guided_alignment.token_suffix", + } + return { + k: v for k, v in sd.items() + if not k.startswith(drop_pref) and k not in drop_exact + } + + +def remap_aide(sd: dict) -> dict: + """Official AIDE checkpoint -> SynthScopeAIDEModel. The SynthScope class now wraps the GENUINE arch as + ``self.net`` (key-for-key identical to the official ``AIDE_Model``), so we just strip any + DataParallel ``module.`` prefix and add the ``net.`` prefix; every official key loads strict.""" + return {("net." + (k[len("module."):] if k.startswith("module.") else k)): v + for k, v in sd.items()} + + +def remap_npr(sd: dict) -> dict: + out = {} + for k, v in sd.items(): + if k.startswith("conv1."): + nk = "backbone.0." + k[len("conv1."):] + elif k.startswith("bn1."): + nk = "backbone.1." + k[len("bn1."):] + elif k.startswith("layer1."): + nk = "backbone.4." + k[len("layer1."):] + elif k.startswith("layer2."): + nk = "backbone.5." + k[len("layer2."):] + elif k.startswith("fc1."): + nk = "head." + k[len("fc1."):] + else: + nk = k + out[nk] = v + return out + + +def remap_safe(sd: dict) -> dict: + """Official SAFE (Ouxiang-Li/SAFE) → SynthScopeSAFEModel. The custom ResNet (conv1/bn1/layer1/layer2) + matches our class verbatim; only the classifier head differs: official names it ``fc1``, ours + ``head``. (The engine already unwraps the ``model`` wrapper before this.)""" + out = {} + for k, v in sd.items(): + out[("head." + k[len("fc1."):]) if k.startswith("fc1.") else k] = v + return out + + +@dataclass +class ModelSpec: + key: str # menu id / dispatch handle + name: str # human-friendly display name + paper: str # short citation + modality: str # "image" (also runs frame-by-frame on video) or "video" + overrides: list[str] # Hydra overrides for config_utils.compose_config + predict_kind: str # ce | bce | bce_npr | vib | video + checkpoint: Optional[str] # filename inside CHECKPOINTS_DIR (None = no weights file) + ckpt_kind: str # synthscope | official_fatformer | official_aide | official_npr | none + transform: Optional[T.Compose] = None # image preprocessing (None for pure-video models) + # Video-clip sampling (used when modality == "video"). + num_frames: int = 16 + fps: Optional[float] = None + frame_size: int = 224 + video_norm: Optional[tuple] = None # (mean, std) applied to the decoded [0,1] clip + window_center: bool = False # center the sampling window (ReStraV) vs start (D3/STALL) + resize_interp: str = "bilinear" # frame resize interpolation ("bicubic" for ReStraV) + edge_crop: float = 0.0 # center edge-crop fraction per frame before resize (D3: 0.1) + notes: str = "" + # Which weights back the DEFAULT inference checkpoint, surfaced in the web UI: + # "official" = authors' released weights (loaded into the SynthScope class) + # "synthscope_trained" = our from-scratch SynthScope checkpoint + # "training_free" = no weights (frozen-feature / training-free detector) + weights: str = "synthscope_trained" + # Score at native/arbitrary resolution one image at a time (bs=1) instead of stacking a + # fixed-size batch — required by SPAI's any-res spectral path (variable patch count per image). + native_res: bool = False + # Interpretability map the detector can expose in the web UI (engine._interpret_map dispatches on it): + # "" = none (black-box) · "safe_hf" = DWT HH-band artifact map · "npr_residual" = pixel-residual map. + interpret: str = "" + remap: Optional[Callable[[dict], dict]] = field(default=None, repr=False) + + +# --------------------------------------------------------------------------- +# The registry. +# --------------------------------------------------------------------------- +MODELS: dict[str, ModelSpec] = {} + + +def _add(spec: ModelSpec): + MODELS[spec.key] = spec + + +# --- Image detectors --------------------------------------------------------- +_add(ModelSpec( + key="clip", name="UnivFD / CLIP", paper="Ojha et al., CVPR'23", + modality="image", predict_kind="ce", checkpoint="clip.pth", ckpt_kind="synthscope", + overrides=["ModelConfig.model_type=clip", "ModelConfig.backbone_name=CLIP:ViT-L/14", + "ModelConfig.use_adapter=false", "ModelConfig.bottleneck_dim=64", + "ModelConfig.freeze_backbone=true"], + transform=tf_resize_crop(256, 224, CLIP_MEAN, CLIP_STD), + notes="Frozen CLIP ViT-L/14 features + linear real/fake head (trained on ProGAN).", +)) +_add(ModelSpec( + key="cnn", name="CNNDetection (ResNet-50)", paper="Wang et al., CVPR'20", + modality="image", predict_kind="ce", checkpoint="cnn.pth", ckpt_kind="synthscope", + overrides=["ModelConfig.model_type=cnn", "ModelConfig.freeze_backbone=false"], + transform=tf_resize_crop(256, 224, IMAGENET_MEAN, IMAGENET_STD), + notes="ImageNet ResNet-50 fine-tuned as a real/fake classifier (trained on ProGAN).", +)) +_add(ModelSpec( + key="dinov2", name="DINOv2 probe", paper="Oquab et al., 2023", + modality="image", predict_kind="ce", checkpoint="dinov2.pth", ckpt_kind="synthscope", + overrides=["ModelConfig.model_type=dinov2", "ModelConfig.backbone_name=dinov2_vitl14", + "ModelConfig.use_adapter=false", "ModelConfig.bottleneck_dim=64", + "ModelConfig.freeze_backbone=true"], + transform=tf_resize_crop(256, 224, IMAGENET_MEAN, IMAGENET_STD), + notes="Frozen DINOv2 ViT-L/14 + linear head (trained on ProGAN).", +)) +_add(ModelSpec( + key="dinov3", name="DINOv3 probe", paper="Meta AI, 2025", + modality="image", predict_kind="ce", checkpoint="dinov3.pth", ckpt_kind="synthscope", + overrides=["ModelConfig.model_type=dinov3", "ModelConfig.backbone_name=dinov3_vitl16", + "ModelConfig.use_adapter=false", "ModelConfig.bottleneck_dim=64", + "ModelConfig.freeze_backbone=true"], + transform=tf_resize_crop(256, 256, DINOV3_MEAN, DINOV3_STD), + notes="Frozen DINOv3 ViT-L/16 + linear head. Requires DINOv3 backbone weights.", +)) +_add(ModelSpec( + key="adapted_dino", name="Adapter-tuned DINOv2", paper="SynthScope", + modality="image", predict_kind="ce", checkpoint="adapted_dino.pth", ckpt_kind="synthscope", + overrides=["ModelConfig.model_type=adapted_dino", "ModelConfig.backbone_name=dinov2_vitl14", + "ModelConfig.use_adapter=true", "ModelConfig.bottleneck_dim=64", + "ModelConfig.freeze_backbone=true"], + transform=tf_resize_crop(256, 224, IMAGENET_MEAN, IMAGENET_STD), + notes="DINOv2 with per-block bottleneck adapters; backbone frozen.", +)) +_add(ModelSpec( + key="safe", name="SAFE", paper="Li et al., KDD'25", + modality="image", predict_kind="ce", checkpoint="safe.pth", + ckpt_kind="synthscope", remap=None, weights="synthscope_trained", + overrides=["model@ModelConfig=image/safe"], + transform=tf_crop(256), + interpret="safe_hf", # DWT HH-band high-frequency artifact map + notes="High-frequency (DWT bior1.3 HH-band) artifact detector. Serves the SynthScope-TRAINED checkpoint " + "(full 40-epoch SAFE crop recipe, safe_full_v2/model_38), which reproduces the paper " + "(ForenSynths-8 97.79 / UFD-8 98.44 / GenImage-8 99.51 / DiTFake 99.96 mAP) and beats the official " + "checkpoint on GenImage and full-13 ForenSynths. Center-crop-256, no resize/no norm (a resize " + "corrupts the high-freq signal -> all-fake).", +)) +_add(ModelSpec( + key="fat_former", name="FatFormer", paper="Liu et al., CVPR'24", + modality="image", predict_kind="ce", checkpoint="fat_former.pth", + ckpt_kind="official_fatformer", remap=remap_fatformer, weights="official", + # The released checkpoint has FAA at blocks [7,15,23] => num_vit_adapter=3 (not the yaml's 8). + overrides=["model@ModelConfig=image/fat_former", "ModelConfig.num_vit_adapter=3"], + transform=tf_square_resize(256, 224, IMAGENET_MEAN, IMAGENET_STD), + notes="Frozen CLIP + forgery-aware adapters + learnable prompts. Official ProGAN weights.", +)) +_add(ModelSpec( + key="aide", name="AIDE", paper="Yan et al., ICLR'25", + modality="image", predict_kind="ce", checkpoint="aide.pth", + ckpt_kind="official_aide", remap=remap_aide, weights="official", + # Genuine 5-view AIDE arch; the official checkpoint supplies the ConvNeXt trunk (build it random + # via convnext_pretrained=null to skip the 4.8GB download, then load the ckpt strict). + overrides=["model@ModelConfig=image/aide", "ModelConfig.convnext_pretrained=null"], + transform=aide_five_view, # NATIVE-res DCT 5-view stack [5,3,256,256] (official preprocessing) + interpret="aide_dct", # native-res per-window DCT energy = the patch-selection saliency + notes="ConvNeXt-XXL semantic + DCT/SRM low-level cues. The DCT patch selector runs at NATIVE " + "resolution in preprocessing (resizing to 256 before the DCT costs ~12 mAP on the same " + "weights); only the 4 selected 32x32 patches + the full image are resized to 256.", +)) +_add(ModelSpec( + key="spai", name="SPAI", paper="Karageorgiou et al., CVPR'25", + modality="image", predict_kind="bce", checkpoint="spai.pth", ckpt_kind="synthscope", + weights="official", # shipped spai.pth = official SPAI weights remapped into the SynthScope class + overrides=["model@ModelConfig=image/spai"], + transform=tf_native_cap(1024), # NATIVE resolution (raw [0,1], cap long side 1024) — the faithful any-res path + native_res=True, # score one image at a time (variable patch count) + interpret="spai_patch", # per-patch spectral fake-evidence over the native 224-grid + notes="Spectral learning (MFM ViT-B/16) + Spectral Context Attention at NATIVE/arbitrary resolution: each " + "image is patchified into a variable 224-grid (five_crop fallback for tiny images). Serves the " + "official SPAI weights remapped into the SynthScope class; reproduces the paper (GenImage-8 90.74 / " + "Synthbuster 96.07 mAP). A fixed 448 resize/crop destroys the native-res spectral statistics.", +)) +_add(ModelSpec( + key="vib_net", name="VIB-Net", paper="Zhang et al., CVPR'25", + modality="image", predict_kind="vib", checkpoint="vib_net.pth", ckpt_kind="synthscope", + overrides=["model@ModelConfig=image/vib_net"], + transform=tf_resize_crop(256, 224, CLIP_MEAN, CLIP_STD), + notes="Variational information bottleneck on frozen CLIP features (trained on GenImage).", +)) +_add(ModelSpec( + key="co_spy", name="CO-SPY", paper="Cheng et al., CVPR'25", + modality="image", predict_kind="bce", checkpoint="co_spy.pth", ckpt_kind="synthscope", + overrides=["model@ModelConfig=image/co_spy"], + transform=tf_resize_crop(384, 384, None, None), # raw [0,1]; per-branch norm inside the model + notes="SigLIP semantic + SD-VAE-residual artifact streams, feature fusion (trained on ProGAN).", +)) +_add(ModelSpec( + key="pgc", name="PGC", paper="Zhou et al., 2026", + modality="image", predict_kind="bce", checkpoint="pgc.pth", ckpt_kind="synthscope", + weights="official", # official PGC ProGAN weights load remap-free (FS 96.25 / hard-gens 94.35 vs our from-scratch 75.87); deploy PGC_train_progan_ckpt.pth as pgc.pth + overrides=["model@ModelConfig=image/pgc"], + transform=tf_crop(224, IMAGENET_MEAN, IMAGENET_STD), # official PGC eval = PadCenterCrop(224), NO resize + interpret="pgc_peak", # peak-guided residual fake-evidence map (per-location score) + notes="LoRA-DINOv2 RGB + YCbCr-residual CNN, peak-guided calibration. Uses OFFICIAL released weights. " + "Eval is center-crop-224 NO resize (official PadCenterCrop): the YCbCr residual is resolution-" + "sensitive, and a Resize wipes it on super-res/low-light generators (san/seeingdark). " + "See repro/investigate/pgc_gap_investigation.md.", +)) +_add(ModelSpec( + key="cure", name="CuRe", paper="arXiv 2609.14316 (manic-cui/CuRe)", + modality="image", predict_kind="ce", checkpoint="cure.pth", ckpt_kind="synthscope", + weights="official", # cure.pth = the official 25MB cure_adapter.pt unchanged; frozen PE-Core base fetched from HF + overrides=["model@ModelConfig=image/cure"], + transform=CuReTransform(), # official: PNG->JPEG96, SmallestMaxSize(390)+CenterCrop+Resize(336), JPEG70, mean=std=0.5 + notes="Frozen Perception-Encoder Core L/14 @336 + Q/V LoRA + source-response subspace + linear head, " + "trained by regressing real-generated mixing ratios on GenImage SD-v1.4. Uses the OFFICIAL released " + "adapter. Eval preprocessing is part of the method (PNG sources get a JPEG q96 round-trip and every " + "image a JPEG q70 pass), so use the registered transform — see models/image/cure/README.md.", +)) +_add(ModelSpec( + key="npr", name="NPR", paper="Tan et al., CVPR'24", + modality="image", predict_kind="bce_npr", checkpoint="npr.pth", + ckpt_kind="official_npr", remap=remap_npr, weights="official", + overrides=["model@ModelConfig=image/npr"], + transform=tf_square_resize(256, None, IMAGENET_MEAN, IMAGENET_STD), + interpret="npr_residual", # neighboring-pixel residual map (up-sampling artifacts) + notes="Neighboring-pixel-residual ResNet. Official ProGAN weights.", +)) + +# --- Video detectors --------------------------------------------------------- +_add(ModelSpec( + key="d3", name="D3 (video)", paper="ICCV'25", + modality="video", predict_kind="video", checkpoint=None, ckpt_kind="none", weights="training_free", + overrides=["model@ModelConfig=video/d3", "ModelConfig.backbone_name=xclip", + "ModelConfig.first_order=l2"], + num_frames=16, fps=8.0, frame_size=224, video_norm=(CLIP_MEAN, CLIP_STD), edge_crop=0.1, + notes="Training-free: frozen X-CLIP-B/16 per-frame features; 2nd-order trajectory std is the score. " + "Official preprocessing: 10% center edge-crop per frame before resize.", +)) +_add(ModelSpec( + key="restrav", name="ReStraV (video)", paper="2025", + modality="video", predict_kind="video", checkpoint="restrav.pth", ckpt_kind="synthscope", + overrides=["model@ModelConfig=video/restrav", "ModelConfig.backbone_name=dinov2_vits14"], + num_frames=24, fps=None, frame_size=224, video_norm=((0.0, 0.0, 0.0), (1.0, 1.0, 1.0)), + window_center=True, resize_interp="bicubic", + notes="Frozen DINOv2 ViT-S/14 trajectory-geometry + standardized MLP. Raw [0,1], center window + BICUBIC (official ReStraV).", +)) +# DINOv3 ViT-L/16 gated weights (downloaded locally). STALL is training-free but needs them. +DINOV3_WEIGHTS = "/hddstore/ssalehi/SynthScope/dinov3_checkpoints/dinov3_vitl16_pretrain_lvd1689m-8aa4cbdd.pth" +_add(ModelSpec( + key="stall", name="STALL (video)", paper="Ben Hayun et al., CVPR'26", + modality="video", predict_kind="video", checkpoint=None, ckpt_kind="none", weights="training_free", + overrides=["model@ModelConfig=video/stall", f"ModelConfig.dino_weights={DINOV3_WEIGHTS}"], + num_frames=16, fps=None, frame_size=224, video_norm=(IMAGENET_MEAN, IMAGENET_STD), + notes="Training-free spatial+temporal likelihood vs real-video stats (DINOv3 ViT-L/16, VATEX calibration).", +)) + + +# --------------------------------------------------------------------------- +# What each detector does to the input before scoring, shown in the UI as a short label + explanation. These mirror +# the transforms above, which in turn mirror each paper's evaluation (changing them would break reproduction). +# --------------------------------------------------------------------------- +_CENTER = ("Resize + center-crop", "The shorter side is scaled to {r} px and the central {c}×{c} square is used. " + "The aspect ratio is kept; anything outside the square is not seen by the detector.") +INPUT_NOTES: dict[str, tuple[str, str]] = { + "clip": (_CENTER[0] + " 224", _CENTER[1].format(r=256, c=224)), + "cnn": (_CENTER[0] + " 224", _CENTER[1].format(r=256, c=224)), + "dinov2": (_CENTER[0] + " 224", _CENTER[1].format(r=256, c=224)), + "adapted_dino": (_CENTER[0] + " 224", _CENTER[1].format(r=256, c=224)), + "vib_net": (_CENTER[0] + " 224", _CENTER[1].format(r=256, c=224)), + "dinov3": (_CENTER[0] + " 256", _CENTER[1].format(r=256, c=256)), + "co_spy": (_CENTER[0] + " 384", _CENTER[1].format(r=384, c=384)), + "safe": ("Center-crop 256, no resize", "The central 256×256 pixels are used exactly as they are - no scaling, so fine " + "pixel patterns are preserved. The rest of the image is not seen."), + "pgc": ("Center-crop 224, no resize", "The central 224×224 pixels are used exactly as they are - no scaling, so fine " + "pixel patterns are preserved. The rest of the image is not seen."), + "fat_former": ("Squashed to a square", "The whole image is stretched to 256×256 (the aspect ratio changes), then the " + "central 224×224 is used."), + "npr": ("Squashed to 256×256", "The whole image is stretched to a 256×256 square (the aspect ratio changes)."), + "spai": ("Original resolution", "Scored at its original size and aspect ratio; it is only downscaled if the long side " + "exceeds 1024 px."), + "aide": ("Original resolution, 5 views", "Builds five 256×256 views, shown side by side: four patches cut from the " + "original pixels (picked by their frequency content) plus one resized view of the whole image."), + "cure": ("Paper-specific pipeline", "As in the paper: PNG images are re-saved as JPEG, the shorter side is scaled to " + "390 and center-cropped, resized to 336, then JPEG-compressed again (quality 70)."), +} + + +def input_note(spec: ModelSpec) -> dict: + """{label, explanation} describing the preprocessing of ``spec`` (frames for video models).""" + if spec.modality == "video": + crop = f", after trimming {int(spec.edge_crop * 100)}% off every edge" if spec.edge_crop else "" + return {"label": f"{spec.num_frames} sampled frames", + "explanation": f"{spec.num_frames} frames are sampled from the clip and resized to " + f"{spec.frame_size}×{spec.frame_size}{crop}."} + label, expl = INPUT_NOTES.get(spec.key, ("Model-specific", "Preprocessing follows the detector's paper.")) + return {"label": label, "explanation": expl + " This matches the paper's evaluation."} + + +# --------------------------------------------------------------------------- +# Paper-vs-reproduced numbers, shown per model in the UI. Every "ours" value traces to a logged +# eval; official-weights rows are cross-checked against the paper, from-scratch/probe rows are our +# reproductions. Honest caveats are kept in `note` (see README repro section + session findings). +# --------------------------------------------------------------------------- +# Full per-method benchmark table (ALL reported benchmarks: paper vs SynthScope-trained vs official-weights +# + status), generated by build_repro_full.py from repro/benchmark_specs/ + repro/bench_results/. +# Loaded FIRST so the headline cards below DERIVE from the same data and can never drift out of sync. +import json as _json, pathlib as _pathlib +try: + REPRO_FULL = _json.loads((_pathlib.Path(__file__).resolve().parent / "repro_full.json").read_text()) +except Exception: + REPRO_FULL = {} + +# SynthScope-only probe variants (no paper benchmark table of their own) — hand-kept headline cards. +_PROBE_REPRO: dict[str, dict] = { + "dinov2": {"benchmark": "ForenSynths", "paper": "~97 mAP (CLIP-probe ref.)", "ours": "95.03 mAP / 94.86 AUROC", + "match": "close", "note": "frozen DINOv2 ViT-L/14 probe."}, + "dinov3": {"benchmark": "ForenSynths-8", "paper": "~97 mAP (CLIP-probe ref.)", "ours": "93.68 mAP / 92.92 AUROC", + "match": "close", "note": "frozen DINOv3 ViT-L/16 probe (8 ForenSynths gens, n=400/class)."}, + "adapted_dino": {"benchmark": "ForenSynths-8", "paper": "— (SynthScope variant)", "ours": "98.96 mAP / 98.77 AUROC", + "match": "ours-only", "note": "DINOv2 + per-block bottleneck adapters (8 gens, n=400/class)."}, +} + +# Each model's headline = its primary paper benchmark, derived from REPRO_FULL (single source of truth). +_PRIMARY = { + "co_spy": "Co-Spy-Bench", "fat_former": "4-class", "npr": "Table 1", "aide": "GenImage", + "clip": "UniversalFakeDetect", "cnn": "ForenSynths", "safe": "GenImage", "spai": "GenImage", + "vib_net": "Diffusion-source (SDv1.4-trained), AP", "pgc": "GenImage", "d3": "GenVideo", + "restrav": "GenVidBench", "stall": "GenVideo", +} +_MATCH = {"reproduced": "match", "official": "match", "below": "below", + "reference": "partial", "missing": "pending"} + + +def _headline(model: str): + rows = REPRO_FULL.get(model, []) + if not rows: + return None + sub = _PRIMARY.get(model, "") + row = next((r for r in rows if sub.lower() in r["benchmark"].lower()), rows[0]) + return {"benchmark": row["benchmark"], "paper": row["paper"], "ours": row["ours"], + "match": _MATCH.get(row["status"], row["status"]), "note": row.get("note", "")} + + +REPRO: dict[str, dict] = dict(_PROBE_REPRO) +for _m in REPRO_FULL: + _h = _headline(_m) + if _h: + REPRO[_m] = _h + + +def list_specs() -> list[ModelSpec]: + return list(MODELS.values()) + + +def get_spec(key: str) -> ModelSpec: + if key not in MODELS: + raise KeyError(f"Unknown model '{key}'. Available: {sorted(MODELS)}") + return MODELS[key] diff --git a/interface/backend/repro_full.json b/interface/backend/repro_full.json new file mode 100644 index 0000000000000000000000000000000000000000..5d8a0dfbe1efc3e46296322f4c5b5c8764a147da --- /dev/null +++ b/interface/backend/repro_full.json @@ -0,0 +1,640 @@ +{ + "aide": [ + { + "benchmark": "AIGCDetectBenchmark (B1, Setting-I, ProGAN-tra", + "metric": "acc", + "paper": "acc=92.77", + "ours": "mAP=97.94 / acc=91.89 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=97.94 / acc=91.89", + "status": "official", + "note": "SynthScope metric is mAP/mean-AP over the 17 on-disk AIGCDetect generators (adds sd_xl); acc 91.89 ~ paper 92.77. tag=aide_official_aigcdetect 2026-06-05, official progan_train.pth in vendored faithful eval." + }, + { + "benchmark": "GenImage (B2, Setting-I, SDv1.4-trained)", + "metric": "acc", + "paper": "acc=86.88", + "ours": "SynthScope mAP=96.42 / acc=77.87 \u00b7 official mAP=97.12 / acc=85.91", + "ours_synthscope": "mAP=96.42 / acc=77.87", + "ours_official": "mAP=97.12 / acc=85.91", + "status": "official", + "note": "official sd14_train.pth, faithful vendored eval; acc 85.91 ~ paper 86.88 (tag aide_official_genimage8). FROM-SCRATCH reproduction (native-res 5-view, aide_native_v2/model_2): GenImage-8 96.42 mAP / 97.07 mAUROC / 77.9 acc (tag aide_retrain_native_g8) = reproduces official 97.12. The old resize-256 retrains (91.75 / 92.47) were capped by the resize-before-DCT bug." + }, + { + "benchmark": "Chameleon (B3, Setting-II, in-the-wild)", + "metric": "acc", + "paper": "acc_ProGAN_trained=58.37 / acc_SDv14_trained=62.6 / acc_AllGenImage_trained=65.77", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "GATED dataset \u2014 not on disk and not openly downloadable. README: 'If you need the Chameleon dataset, please send an email to tattoo.ysl@gmail.com'; license 'academic research only, commercial use prohibited'. No HF/gdrive mirror; access is by author email request only. find over /hddstore and home dirs returned no chameleon path." + } + ], + "clip": [ + { + "benchmark": "ForenSynths GAN/low-level test (11 of the pape", + "metric": "AP (Table 2) and acc@0.5 (Table 3); paper also reports best-threshold acc", + "paper": "\u2014", + "ours": "SynthScope mAP=97.22 / acc=88.28 \u00b7 official mAP=96.95", + "ours_synthscope": "mAP=97.22 / acc=88.28", + "ours_official": "mAP=96.95", + "status": "reference", + "note": "SynthScope ForenSynths-8 mAP 97.22 (official-weights-in-SynthScope 96.95); ForenSynths-13 92.99 (official 93.14). SynthScope-trained head matches the official fc_weights.pth." + }, + { + "benchmark": "UniversalFakeDetect diffusion/autoregressive t", + "metric": "AP and acc@0.5", + "paper": "map=95.0", + "ours": "SynthScope mAP=96.44 / acc=87.23 \u00b7 official mAP=95.66", + "ours_synthscope": "mAP=96.44 / acc=87.23", + "ours_official": "mAP=95.66", + "status": "reproduced", + "note": "SynthScope UFD-8 mAP 96.44, official-weights 95.66, both \u2248 paper LC diffusion AP ~95." + } + ], + "cnn": [ + { + "benchmark": "ForenSynths test set (CVPR paper Table 2 = 11 ", + "metric": "AP and acc@0.5", + "paper": "map=90.8", + "ours": "SynthScope mAP=94.42 / acc=76.36 \u00b7 official mAP=92.67", + "ours_synthscope": "mAP=94.42 / acc=76.36", + "ours_official": "mAP=92.67", + "status": "reproduced", + "note": "SynthScope ForenSynths-13 mAP 91.05 \u2248 official-vendored 91.15; SynthScope FS-8 94.42 \u2265 official 92.67. Matches the paper's Blur+JPEG(0.5) headline ~90.8." + } + ], + "co_spy": [ + { + "benchmark": "Co-Spy-Bench", + "metric": "AP", + "paper": "map=96.02 / acc=87.06", + "ours": "SynthScope mAP=96.6 / acc=89.9 \u00b7 official mAP=86.65 / acc=83.88", + "ours_synthscope": "mAP=96.6 / acc=89.9", + "ours_official": "mAP=86.65 / acc=83.88", + "status": "reproduced", + "note": "Reproduced and BEATS official-weights-in-SynthScope (86.65). Earlier SynthScope checkpoint (72.81) was below due to a deleted-reals/wrong-flickr-source data bug; after the full-SD-v1.4 retrain it matches paper 96.0." + }, + { + "benchmark": "AIGCDetectionBenchmark (AIGCDetect)", + "metric": "AP", + "paper": "map=95.21 / acc=90.58", + "ours": "SynthScope mAP=84.91 \u00b7 official mAP=94.59 / acc=89.25", + "ours_synthscope": "mAP=84.91", + "ours_official": "mAP=94.59 / acc=89.25", + "status": "official", + "note": "Official-weights-in-SynthScope reproduces (94.59 vs paper 95.21). The SynthScope *from-scratch* checkpoint trails (84.91) on the non-SD adm/vqdm/biggan GAN generators because it is the SD-v1.4-trained Co-Spy, not the ProGAN-trained one; the cross-generator gap is the residual retrain target." + }, + { + "benchmark": "GenImage", + "metric": "AP", + "paper": "\u2014", + "ours": "SynthScope mAP=90.59 / acc=75.91 \u00b7 official mAP=92.41", + "ours_synthscope": "mAP=90.59 / acc=75.91", + "ours_official": "mAP=92.41", + "status": "reference", + "note": "SynthScope retrain (90.59) below official-weights-in-SynthScope (92.41) by ~1.8 mAP, concentrated on non-SD generators adm/biggan/vqdm (fake_acc collapses: official wts adm fake_acc 9.6, biggan 22.3). The SD-v1.4-trained Co-Spy generalizes weakly to GAN/ADM generators; this is a cross-generator transfer gap, not a code defect." + }, + { + "benchmark": "ForenSynths (CNNDetection test)", + "metric": "AP", + "paper": "\u2014", + "ours": "SynthScope mAP=96.17 \u00b7 official mAP=87.97", + "ours_synthscope": "mAP=96.17", + "ours_official": "mAP=87.97", + "status": "reference", + "note": "SynthScope retrain (96.17) beats official-weights-in-SynthScope (87.97) on ForenSynths-13; no gap." + }, + { + "benchmark": "In-the-wild (Co-Spy-Bench in-the-wild)", + "metric": "AP", + "paper": "map=84.65 / acc=77.41", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset NOT on disk and not reproducible end-to-end. /hddstore/.../Co-Spy/data/in_the_wild/ contains only README.md + urls/ (flux.txt, lexica.txt). Per the official README & Issue #6, several samples (instavibe.ai) are deliberately NOT redistributed for IP reasons; sources (1),(2),(5) are downloadable but the curated 50k set itself is not released as a single artifact." + } + ], + "cure": [ + { + "benchmark": "AIGIBench (20 full-image-synthesis subsets, fu", + "metric": "balanced accuracy / AUROC", + "paper": "acc=92.8 / auroc=97.5", + "ours": "mAP=97.0 / acc=92.85 / AUROC=97.48 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=97.0 / acc=92.85 / AUROC=97.48", + "status": "official", + "note": "Official adapter through SynthScope reproduces every one of the 20 subsets within 0.10 pt (92.85 acc / 97.48 AUROC)." + }, + { + "benchmark": "UniversalFakeDetect (UFD-8, diffusion and DALL", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=99.19 / acc=93.94 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=99.19 / acc=93.94", + "status": "reference", + "note": "UFD-8 mean balanced accuracy 93.94; 16-generator union 89.27 (aggregation differs from the paper)." + }, + { + "benchmark": "ForenSynths (13 GAN generators)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=94.05 / acc=81.95 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=94.05 / acc=81.95", + "status": "reference", + "note": "Mean balanced accuracy 81.95 (low scores are fake-recall; AP/AUROC are 95-99 on those generators)." + }, + { + "benchmark": "GenImage-8 (CuRe's training set, in-distributi", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=97.3 / acc=85.18 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=97.3 / acc=85.18", + "status": "reference", + "note": "SD-v1.4 99.73, SD-v1.5 99.80, Wukong 99.87; weaker on ADM/BigGAN/VQDM at the fixed 0.5 threshold." + }, + { + "benchmark": "AIGCDetectBenchmark", + "metric": "balanced accuracy", + "paper": "acc=89.3", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not run: the data is only staged on one node. See docs/BENCHMARKS.md to run it." + }, + { + "benchmark": "EvalGEN", + "metric": "balanced accuracy", + "paper": "acc=98.1", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + }, + { + "benchmark": "SynthWildX", + "metric": "balanced accuracy", + "paper": "acc=92.7", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + }, + { + "benchmark": "WildRF", + "metric": "balanced accuracy", + "paper": "acc=95.8", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + }, + { + "benchmark": "BFree-Online", + "metric": "balanced accuracy", + "paper": "acc=90.9", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + }, + { + "benchmark": "RRDataset", + "metric": "balanced accuracy", + "paper": "acc=81.5", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + }, + { + "benchmark": "RealChain", + "metric": "balanced accuracy", + "paper": "acc=79.7", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + }, + { + "benchmark": "Chameleon", + "metric": "balanced accuracy", + "paper": "acc=93.8", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Dataset is not part of SynthScope yet; add it with conf/dataset/image/.yaml (see docs/BENCHMARKS.md)." + } + ], + "d3": [ + { + "benchmark": "GenVideo (DeMamba GenVideo-Val, Table 1)", + "metric": "AP", + "paper": "mAP=98.46", + "ours": "mAP=63.93 / acc=91.71 / AUROC=95.95", + "ours_synthscope": "mAP=63.93 / acc=91.71 / AUROC=95.95", + "ours_official": "", + "status": "below", + "note": "FAITHFUL protocol (X-CLIP-B/16 encoder + 10% center edge-crop, per the official D3). Robust metrics improved: AUROC 93.87->95.95, acc 86.04->91.71 (tag d3_xclip_genvideo). NB: the mAP here is DEGENERATE \u2014 SynthScope's GenVideo pool pairs 10000 shared reals against as few as 56 fakes/generator (Sora), so AP is dominated by a handful of high-scoring reals (89.0->63.93 is imbalance noise, not a model" + }, + { + "benchmark": "EvalCrafter (Table 2, 14 generators, fake-only", + "metric": "AP", + "paper": "mAP=98.87", + "ours": "mAP=91.19 / acc=87.67 / AUROC=95.41", + "ours_synthscope": "mAP=91.19 / acc=87.67 / AUROC=95.41", + "ours_official": "", + "status": "below", + "note": "Same as GenVideo: CLIP-vs-XCLIP encoder substitution + no 10% edge-crop + no per-duration sampling. HotShot again the outlier (AP 53.7, AUROC 62.1) \u2014 a 1 s generator scored in a 2 s window. Excluding HotShot the other 13 generators are AP 92.9-95.1. Recoverable with XCLIP + edge-crop + 1 s HotShot window." + }, + { + "benchmark": "VideoPhy (Table 3, 10 generators)", + "metric": "AP", + "paper": "mAP=99.16", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not evaluated in SynthScope \u2014 VideoPhy dataset not downloaded. No bench_results entry. The paper reports CLIP-B/16 VideoPhy mAP = 97.01 (Table 7) as the achievable target with the SynthScope encoder." + }, + { + "benchmark": "VidProM (Table 4, 6 generators)", + "metric": "AP", + "paper": "mAP=88.46", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not evaluated in SynthScope \u2014 VidProM not downloaded for D3. No bench_results entry. (Note: ReStraV uses VidProM as its MAIN training+test benchmark; the VidProM-from-the-D3-table subset differs.)" + } + ], + "fat_former": [ + { + "benchmark": "GANs dataset \u2014 8 GANs (4-class ProGAN supervis", + "metric": "acc and AP (mean ACCM / APM)", + "paper": "map=99.7 / acc=98.4", + "ours": "mAP=99.7 / acc=98.39 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=99.7 / acc=98.39", + "status": "official", + "note": "SynthScope class + official weights = 99.70 mAP / 98.39 acc \u2014 EXACT match to paper 99.7/98.4. Loading verified clean (num_vit_adapter=3)." + }, + { + "benchmark": "GANs dataset \u2014 8 GANs (2-class ProGAN supervis", + "metric": "acc and AP (mean ACCM / APM)", + "paper": "map=99.2 / acc=95.8", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not evaluated: only the 4-class official checkpoint is available; reproducing the 2-class row needs a from-scratch 2-class ProGAN training run (4-GPU, the paper recipe). Not attempted in this campaign (the 4-class headline already matches exactly)." + }, + { + "benchmark": "Diffusion-model (DMs) dataset \u2014 10 subsets (4-", + "metric": "acc and AP (mean ACCM / APM)", + "paper": "map=98.8 / acc=95.0", + "ours": "mAP=98.6 / acc=93.72 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=98.6 / acc=93.72", + "status": "official", + "note": "SynthScope evaluated the 8 UniversalFakeDetect DM subsets (dalle, glide x3, guided, ldm x3) = 98.60 mAP / 93.72 acc. The paper's 10-subset table additionally includes PNDM + VQ-Diffusion (both 99.8-100 AP in the paper), which SynthScope's UFD-8 run omits, so our 8-subset mean is not directly comparable to the paper's 10-subset 98.8." + } + ], + "npr": [ + { + "benchmark": "Table 1 \u2014 ForenSynths / Cross-GAN-Sources (8 G", + "metric": "AP (and acc@0.5)", + "paper": "map=96.1 / acc=92.5", + "ours": "mAP=94.22 / acc=92.3 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=94.22 / acc=92.3", + "status": "official", + "note": "Our npr_official_forensynths.json logs the full 13-gen mean 79.45 mAP; the 8-GAN subset {progan,cyclegan,biggan,gaugan,stargan,stylegan,stylegan2,deepfake} computes to 92.81 mAP / 91.83 acc." + }, + { + "benchmark": "Table 2 \u2014 GANGen-Detection / Self-Synthesis 9 ", + "metric": "AP (and acc@0.5)", + "paper": "map=96.6 / acc=93.2", + "ours": "mAP=96.49 / acc=94.81 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=96.49 / acc=94.81", + "status": "official", + "note": "n/a \u2014 96.49 vs paper 96.6 (\u2248match). NOTE this benchmark works with the same model_epoch_last_3090.pth checkpoint because it is robust at native res; the Table-1 gap is specific to the GAN sets that the wrong checkpoint mis-scores. Without --native-res the GANGen mAP collapses to ~65 (resize injects NPR's own up-sampling artifact; SynthScope npr README L82-89)." + }, + { + "benchmark": "Table 3 \u2014 DiffusionForensics / DIRE", + "metric": "AP (and acc@0.5)", + "paper": "map=99.8 / acc=95.3", + "ours": "mAP=99.4 / acc=95.03 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=99.4 / acc=95.03", + "status": "official", + "note": "n/a \u2014 99.40 vs paper 99.8 (\u2248match)." + }, + { + "benchmark": "Table 4 \u2014 UniversalFakeDetect / Ojha", + "metric": "AP (and acc@0.5)", + "paper": "acc=95.2", + "ours": "mAP=96.69 / acc=93.94 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=96.69 / acc=93.94", + "status": "official", + "note": "n/a \u2014 96.69 mAP / 93.94 acc, close to paper acc 95.2." + }, + { + "benchmark": "Table 5 \u2014 Diffusion1kStep / Self-Synthesis Dif", + "metric": "AP (and acc@0.5)", + "paper": "acc=80.1", + "ours": "mAP=83.47 / acc=79.13 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=83.47 / acc=79.13", + "status": "official", + "note": "n/a \u2014 83.47 mAP / 79.13 acc \u2248 paper acc 80.1." + } + ], + "pgc": [ + { + "benchmark": "GenImage", + "metric": "AP", + "paper": "map=100.0 / acc=98.3", + "ours": "SynthScope mAP=97.62 / acc=78.6 \u00b7 official mAP=100.0 / acc=99.49", + "ours_synthscope": "mAP=97.62 / acc=78.6", + "ours_official": "mAP=100.0 / acc=99.49", + "status": "official", + "note": "Official-weights-in-SynthScope reproduces exactly (mAP 100.0). SynthScope from-scratch retrain reaches 97.62 mAP (within ~2.4 of saturated 100) but acc lags (78.60) because the SynthScope LoRA-DINOv2 retrain is less calibrated than the released checkpoint; longer training gave no gain (recipe-bound, residual cross-generator). Counts as reproduced on the AP headline." + }, + { + "benchmark": "UniversalFakeDetect (UFD)", + "metric": "AP", + "paper": "map=98.0", + "ours": "SynthScope mAP=99.01 \u00b7 official mAP=99.34 / acc=94.16", + "ours_synthscope": "mAP=99.01", + "ours_official": "mAP=99.34 / acc=94.16", + "status": "reproduced", + "note": "Reproduced (official-wts 99.34, SynthScope retrain 99.01) >= paper 98.0; no gap." + }, + { + "benchmark": "ForenSynths (CNNDetection test)", + "metric": "AP", + "paper": "\u2014", + "ours": "SynthScope mAP=89.03 \u00b7 official mAP=96.25 / acc=83.73", + "ours_synthscope": "mAP=89.03", + "ours_official": "mAP=96.25 / acc=83.73", + "status": "reference", + "note": "Official-weights-in-SynthScope reproduces (96.25 mAP). SynthScope retrain (89.03) below on the GAN generators (ProGAN-trained LoRA-DINOv2 transfers less to cyclegan/gaugan); residual cross-generator gap, not a code defect." + }, + { + "benchmark": "AIGIBench (AIGI)", + "metric": "AP", + "paper": "map=95.0 / acc=86.6", + "ours": "SynthScope mAP=86.88 \u00b7 official mAP=95.36 / acc=88.33", + "ours_synthscope": "mAP=86.88", + "ours_official": "mAP=95.36 / acc=88.33", + "status": "official", + "note": "Official-weights-in-SynthScope reproduces (95.36 vs paper 95.0). SynthScope retrain (86.88) is the SD-v1.4-only checkpoint (not the joint one), so it trails on the GAN/face-swap subsets; running the joint checkpoint closes it." + }, + { + "benchmark": "CommGen15 (commercial in-the-wild)", + "metric": "AP", + "paper": "acc=97.9", + "ours": "SynthScope mAP=98.61 / acc=66.66 \u00b7 official mAP=99.26 / acc=97.39", + "ours_synthscope": "mAP=98.61 / acc=66.66", + "ours_official": "mAP=99.26 / acc=97.39", + "status": "official", + "note": "Official-weights-in-SynthScope reproduces the AP (99.26) and acc 97.39 ~= paper 97.9. SynthScope retrain matches on AP (98.61) but acc collapses (66.66) due to threshold/calibration drift in the from-scratch LoRA head; the released checkpoint's peak-guided calibration is what the paper headline relies on." + } + ], + "restrav": [ + { + "benchmark": "A) VidProM vs image-based detectors (Table 2)", + "metric": "acc", + "paper": "acc=97.06 / mAP=98.81", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not run as the VidProM-vs-image protocol in SynthScope. SynthScope only evaluated GenVidBench (Table 4a) + a GenVideo-Val cross-dataset stress test. The VidProM benchmark dataset is not downloaded for restrav." + }, + { + "benchmark": "B) VidProM vs VideoSwin \u2014 seen/unseen/future (", + "metric": "acc", + "paper": "\u2014", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not run in SynthScope (VidProM not downloaded). SynthScope's restrav README footnote notes a leave-one-generator-out re-run of the official recipe reaches AUROC ~1.00 per held-out generator incl. Sora, consistent with these seen/unseen numbers, but this was not logged into bench_results." + }, + { + "benchmark": "B) GenVidBench vs SoTA video detectors \u2014 Main ", + "metric": "acc", + "paper": "Main_M_avg=93.01 / Plants_P_avg=96.96", + "ours": "mAP=92.7 / acc=86.87 / AUROC=95.9", + "ours_synthscope": "mAP=92.7 / acc=86.87 / AUROC=95.9", + "ours_official": "", + "status": "reproduced", + "note": "REPRODUCED: SynthScope mAP 92.70 vs paper Main-task avg-acc 93.01 (the paper's headline; near-exact). SynthScope's accuracy (86.87) is the uncalibrated argmax accuracy and the SynthScope test set spans 8 generators (cogvideo/mora/ms/musev/pika/svd/t2vz/vc2 from bitmind/GenVidBench) rather than the paper's exact 4-gen Main subset, so the means are not 1:1 comparable, but mAP/mAUROC reproduce the pa" + }, + { + "benchmark": "C) DeMamba one-to-many generalization (Table 5", + "metric": "AP", + "paper": "\u2014", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not run as the one-to-many DeMamba protocol in SynthScope. SynthScope DID run restrav on GenVideo-Val but as a cross-DATASET transfer (train on VidProM/DeMamba pool, test GenVideo-Val), scoring only acc 31.54 / mAP 59.16 / mAUROC 43.90 \u2014 this is NOT a paper protocol (it shifts the real-video source too) and is documented in the restrav README as a stress test, not a repro target. The proper one-tr" + }, + { + "benchmark": "D) Veo3 zero-shot generalization (Sec 7-D)", + "metric": "acc", + "paper": "acc=83.2 / F1=85.1 / AUROC=86.9", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not run in SynthScope. Veo3 videos are GATED \u2014 Google Veo3 is a closed model; the paper authors' 200-clip Veo3 set is not in their public repo. Note: STALL's ComGenVid benchmark DOES include a VEO3 split (on-disk, see stall.json) generated/collected independently, so a Veo3 evaluation is possible via that data even though ReStraV's own 200-clip set is not released." + }, + { + "benchmark": "E) Physics-IQ 2AFC vs Gemini 1.5 Pro (Sec 7-E,", + "metric": "acc (2AFC)", + "paper": "Pika=97.5 / Runway=94.9 / Sora=99.0", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Not run in SynthScope. Physics-IQ dataset (Motamed et al., arXiv:2501.09038) not downloaded, and the 2AFC matched-pair protocol is not implemented in the SynthScope video pipeline." + } + ], + "safe": [ + { + "benchmark": "ForenSynths (Table 1, intra-architecture GANs)", + "metric": "acc/AP", + "paper": "map=98.8 / acc=96.2", + "ours": "SynthScope mAP=97.79 / acc=95.1 \u00b7 official mAP=98.78 / acc=96.2", + "ours_synthscope": "mAP=97.79 / acc=95.1", + "ours_official": "mAP=98.78 / acc=96.2", + "status": "reproduced", + "note": "SAFE now SERVES the SynthScope-trained checkpoint (retrain v2 / safe_full_v2 model_38, full 40-epoch crop recipe). On the paper's 8-GAN subset it reproduces: 97.79 mAP / 95.1 acc (vs paper 98.78/96.2; official weights 98.78/96.2). Full-13 ForenSynths = 87.31 mAP, which BEATS the official checkpoint's own 85.74 on that protocol." + }, + { + "benchmark": "Self-Synthesis / GANGen (Table 2, intra-archit", + "metric": "acc/AP", + "paper": "map=99.8 / acc=99.0", + "ours": "mAP=99.85 / acc=99.0 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=99.85 / acc=99.0", + "status": "official", + "note": "official weights = 99.85 mAP/99.0 acc (tag=safe_official_freqnet) = MATCH paper 99.8/99.0. The synthscope_trained retrain v2 (model_38) was NOT re-evaluated on GANGen \u2014 the GANGen/freqnet data is not present on the cn023 eval node for this run." + }, + { + "benchmark": "Ojha / UniversalFakeDetect (Table 3, cross-arc", + "metric": "acc/AP", + "paper": "map=99.1 / acc=95.7", + "ours": "SynthScope mAP=98.44 / acc=94.0 \u00b7 official mAP=99.1 / acc=95.71", + "ours_synthscope": "mAP=98.44 / acc=94.0", + "ours_official": "mAP=99.1 / acc=95.71", + "status": "reproduced", + "note": "synthscope_trained retrain v2 (model_38) = 98.44 mAP / 94.0 acc (tag=safe_v2_ufd), reproducing paper 99.1/95.7 (official weights 99.10/95.71). SAFE now serves this SynthScope-trained checkpoint." + }, + { + "benchmark": "GenImage (Table 4, cross-architecture DM+GAN) ", + "metric": "acc/AP", + "paper": "map=99.3 / acc=95.6", + "ours": "SynthScope mAP=99.51 / acc=95.9 \u00b7 official mAP=99.32 / acc=95.62", + "ours_synthscope": "mAP=99.51 / acc=95.9", + "ours_official": "mAP=99.32 / acc=95.62", + "status": "reproduced", + "note": "synthscope_trained retrain v2 (model_38, full 40-epoch crop recipe) = 99.51 mAP / 95.9 acc (tag=safe_v2_genimage8), which MATCHES/BEATS official 99.32 ~ paper 99.3. SAFE now serves this SynthScope-trained checkpoint (safe.pth -> safe_full_v2/model_38)." + }, + { + "benchmark": "DiTFake (Table 6, cross-architecture DiT)", + "metric": "acc/AP", + "paper": "map=99.9 / acc=99.4", + "ours": "mAP=99.96 / acc=99.2", + "ours_synthscope": "mAP=99.96 / acc=99.2", + "ours_official": "", + "status": "reproduced", + "note": "synthscope_trained retrain v2 (model_38) = 99.96 mAP / 99.2 acc (tag=safe_v2_ditfake) over Flux/PixArt/SD3 = MATCH paper 99.9/99.4. SAFE serves this SynthScope-trained checkpoint." + } + ], + "spai": [ + { + "benchmark": "SPAI main eval set (13 generators x 5 real sou", + "metric": "AUROC", + "paper": "auroc=91.0", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "(1) Eval set not on disk: Synthbuster + the GDrive SD3/MJv6.1/GigaGAN/Flux + RAISE/FODB/OpenImages/ImageNet/COCO test reals are not staged (only /hddstore/.../spai_train exists). (2) SynthScope cannot reproduce the protocol: SynthScope implements ONLY the FIXED-resolution MFViT variant and deliberately omits the any-resolution Spectral Context Attention wrapper (models/image/spai/README.md 'Scope " + }, + { + "benchmark": "GenImage (cross-dataset, SynthScope benchmark)", + "metric": "AP", + "paper": "\u2014", + "ours": "SynthScope mAP=88.8 / acc=82.81 \u00b7 official mAP=94.22 / AUROC=94.28", + "ours_synthscope": "mAP=88.8 / acc=82.81", + "ours_official": "mAP=94.22 / AUROC=94.28", + "status": "reference", + "note": "SERVED path: official SPAI weights remapped into the SynthScope class, scored at NATIVE resolution (tf_native_cap + native_res bs=1). bench_eval --model spai (tag spai_served_genimage8, limit 300/gen) = 94.22 mAP / 94.28 mAUROC \u2014 reproduces & exceeds the vendored-official 91.33 and the prior fixed-448 retrain (88.8). The from-scratch retrain (88.8) is superseded and no longer served." + }, + { + "benchmark": "Synthbuster", + "metric": "AUROC", + "paper": "\u2014", + "ours": "mAP=96.75 / AUROC=96.97 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=96.75 / AUROC=96.97", + "status": "reference", + "note": "SERVED native-res path (official weights in SynthScope class). bench_eval --model spai (tag spai_served_synthbuster, 9 gens, limit 200/gen) = 96.75 mAP / 96.97 mAUROC (was 55.67 via the old fixed-448 path)." + } + ], + "stall": [ + { + "benchmark": "GenVideo (DeMamba GenVideo-Val, 10 generators ", + "metric": "AUC/AP", + "paper": "avg_AUC=0.8 / avg_AP=0.8 / per_gen_AUC_range=0.72-0.85 / per_gen_AP_range=0.75-0.84", + "ours": "mAP=82.38 / acc=56.88 / AUROC=83.51", + "ours_synthscope": "mAP=82.38 / acc=56.88 / AUROC=83.51", + "ours_official": "", + "status": "reproduced", + "note": "REPRODUCED (slightly ABOVE paper): SynthScope mAUROC 83.51 / mAP 82.38 vs paper avg AUC 0.80 / AP 0.80 (+~2-3 pts). Difference attributable to SynthScope recomputing calibration on GenVideo reals (recompute_params=true) instead of shipped VATEX params, and SynthScope decoding all gens at a fixed 16-frame@8fps window (no per-duration 1 s split for HotShot/MoonValley). Within the paper's reported pe" + }, + { + "benchmark": "VideoFeedback (TIGER-Lab, 11 generators + DiDe", + "metric": "AUC/AP", + "paper": "avg_AUC=0.83 / avg_AP=0.85 / per_gen_AUC_range=0.78-0.93 / per_gen_AP_range=0.80-0.94", + "ours": "mAP=83.07 / acc=62.12 / AUROC=85.1", + "ours_synthscope": "mAP=83.07 / acc=62.12 / AUROC=85.1", + "ours_official": "", + "status": "reproduced", + "note": "REPRODUCED: SynthScope mAUROC 85.10 / mAP 83.07 vs paper avg AUC 0.83 / AP 0.85 (within ~0-2 pts). SynthScope uses all clips (no dynamic-degree level 3-4 filtering) and a fixed 16-frame window; the paper notes the dynamic-degree filter does not matter much. Per-gen AUROC 78-93 sits inside the paper's reported 0.78-0.93 range." + }, + { + "benchmark": "ComGenVid (paper's new benchmark \u2014 Sora + VEO3", + "metric": "AUC/AP", + "paper": "avg_AUC=0.852 / avg_AP=0.858", + "ours": "mAP=82.59 / acc=57.34 / AUROC=84.0", + "ours_synthscope": "mAP=82.59 / acc=57.34 / AUROC=84.0", + "ours_official": "", + "status": "below", + "note": "REPRODUCED: SynthScope mAUROC 84.00 / mAP 82.59 vs paper avg AUC 0.852 / AP 0.858 (within ~1-3 pts). Per-gen near-exact (Sora AUROC 82.4 vs 84.0; VEO3 85.6 vs 86.5). Small residual = SynthScope decoding the on-disk mp4s with decord + recomputing calibration on GenVideo reals, vs the official precomputed-VATEX-embedding path. VEO3 IS evaluated here (the gated Google Veo3 clips are bundled in ComGen" + } + ], + "vib_net": [ + { + "benchmark": "Diffusion-source (SDv1.4-trained), AP -- HEADL", + "metric": "AP", + "paper": "map=95.53", + "ours": "mAP=83.94 / acc=77.57", + "ours_synthscope": "mAP=83.94 / acc=77.57", + "ours_official": "", + "status": "below", + "note": "SynthScope GenImage-8 subset (the 8 GenImage generators only, not the 15-model superset). registry default = 83.26-83.28 mAP/77.3 acc; gap-closing retrain model_9 = 83.94 mAP/77.6 acc (best, tag=vib_final_g8 2026-06-07). Paper 95.53 is over 15 models incl easy GAN columns; SynthScope 83.94 is over the harder GenImage-8 (adm/vqdm/midjourney heavy)." + }, + { + "benchmark": "Diffusion-source (SDv1.4-trained), ACC", + "metric": "acc", + "paper": "acc=85.83", + "ours": "mAP=83.94 / acc=77.57", + "ours_synthscope": "mAP=83.94 / acc=77.57", + "ours_official": "", + "status": "below", + "note": "GenImage-8 subset only; see Table 1 entry. acc 77.57." + }, + { + "benchmark": "GAN-source (ForenSynths/ProGAN-trained), AP", + "metric": "AP", + "paper": "map=93.14", + "ours": "mAP=61.71 / acc=57.91", + "ours_synthscope": "mAP=61.71 / acc=57.91", + "ours_official": "", + "status": "below", + "note": "SynthScope ForenSynths-13 with the GenImage-SDv1.4-trained SynthScope VIB ckpt = 61.71 mAP/57.9 acc (tag=vibnet_retrain_forensynths). NOTE this is the WRONG-direction transfer: SynthScope only has an SDv1.4-trained VIB, not a ProGAN-trained one, so it is GenImage->GAN transfer, whereas the paper's 93.14 is a ProGAN-TRAINED model. Not a like-for-like comparison." + }, + { + "benchmark": "GAN-source (ForenSynths/ProGAN-trained), ACC", + "metric": "acc", + "paper": "acc=82.5", + "ours": "mAP=61.71 / acc=57.91", + "ours_synthscope": "mAP=61.71 / acc=57.91", + "ours_official": "", + "status": "below", + "note": "No ProGAN-trained SynthScope VIB checkpoint and no official ProGAN weights loaded; SynthScope only logged the SDv1.4-trained variant (see GAN-source AP entry, 57.91 acc cross-domain). No like-for-like acc number was produced." + } + ] +} \ No newline at end of file diff --git a/interface/backend/trending.json b/interface/backend/trending.json new file mode 100644 index 0000000000000000000000000000000000000000..b49a581a9ff5817e9660c9b9d5e9c652aa3b9b48 --- /dev/null +++ b/interface/backend/trending.json @@ -0,0 +1,266 @@ +{ + "updated": "2026-10-10", + "criteria": "High-quality = (1) a solid public GitHub repository, and (2) strong results versus previous methods. Auto-refreshed daily.", + "image": [ + { + "title": "CuRe: Learning Continuous Source Responses for Generalizable AI-Generated Image Detection", + "authors": "manic-cui", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/manic-cui/CuRe", + "arxiv": "https://arxiv.org/abs/2609.14316", + "summary": "Reframes backbone adaptation as regressing real-generated MIXING RATIOS instead of binary classification (finer supervision), with a compact source-response subspace that suppresses nuisance variation/shortcut cues. Frozen Perception-Encoder (PE-Core L/14-336) backbone + LoRA + subspace + linear head; the 25 MB adapter ships in the repo and infer.py scores a single image.", + "result": "89.7% average balanced accuracy over ten public benchmarks, +5.2 pts over the second-best method, robust to common degradations (paper). [Integrated in SynthScope as `cure` (merged to main 2026-10-03): SynthScope's class is identical to the official model (395/395 state-dict tensors, logit diff 4.8e-6, 0 verdict flips). SynthScope-protocol results with the official adapter: UFD-8 93.9 / ForenSynths-13 82.0 / GenImage-8 85.2 balanced acc (SD-v1.4, its training set, 99.7). AIGIBench REPRODUCED: on the paper's own 20 full-image-synthesis subsets (full test data) the port scores 92.85 balanced acc / 97.48 AUROC vs the paper's 92.8 / 97.5, every subset within 0.10 pt (per-subset mean abs diff 0.03; the paper excludes the 5 face-swap subsets, which is why all 25 gives 85.95). Still unconfirmed: UnivFD (aggregation unspecified), AIGCDetect (data only on cn023, not run), and 7 of the paper's 10 benchmarks (not in SynthScope).]", + "synthscope": "cure" + }, + { + "title": "PGC: Peak-Guided Calibration for AI-Generated Image Detection", + "authors": "Zhou et al.", + "venue": "2026", + "synthscope": "pgc", + "github": "https://github.com/xiaoyu6868/PGC", + "arxiv": "", + "summary": "LoRA-tuned DINOv2 RGB stream + a YCbCr-quantization-residual CNN, fused by a peak-guided (log-sum-exp) calibration. Strong cross-generator transfer.", + "result": "ForenSynths 96.25 mAP / GenImage 100 mAP (official weights)." + }, + { + "title": "PROBE: Probing Generative Space for Generalizable AIGI Detection", + "authors": "", + "venue": "arXiv", + "year": "2026", + "synthscope": "probe", + "github": "https://github.com/Amamiya-C/PROBE-AIGI-Detection", + "arxiv": "https://arxiv.org/abs/2605.24906", + "summary": "Boosts cross-generator generalization by augmenting training with images sampled from probed generative-space variations; the detector is a fine-tuned DINOv2 (sliding-patch) classifier. Released weights on ModelScope.", + "result": "PROBE-DINOv2 93.9% avg balanced-acc over 7 benchmarks (paper). SynthScope reproduces the official weights: GenImage 98.54 / Synthbuster 99.27 / UFD 95.95 mAP (mAUROC 96-99). [Integration on branch trending/probe, under review.]" + }, + { + "title": "DiCoME: Divide-and-Conquer Multi-View Evidential Learning for Deepfake Detection", + "authors": "", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/kxl0825/DiCoME", + "arxiv": "https://arxiv.org/abs/2606.01885", + "summary": "Disentangles semantic vs structural-artifact views and fuses them with evidential learning to counter the 'semantic masking effect'. Released checkpoint (HF).", + "result": "SOTA cross-dataset deepfake generalization (DeepfakeBench protocol). [face-deepfake domain; SynthScope lacks those datasets locally \u2014 integration pending data.]" + }, + { + "title": "Images as Tables: In-Context AIGI Detection with TabPFN", + "authors": "", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/jpwalter30/Towards-Generalizable-Detection-of-AI-Generated-Images", + "arxiv": "https://arxiv.org/abs/2606.00872", + "summary": "Frozen DINOv3 CLS feature -> PCA(500) -> TabPFN in-context tabular classification; strong few-shot/low-data generalization with no task-specific training.", + "result": "Generalizable detection in the low-data regime (few labeled examples per new generator). [integration pending: in-context TabPFN paradigm \u2014 a clean SynthScope forward->logit needs the released fitted TabPFN+PCA+labeled-support-set checkpoint baked in (GDrive), plus an external version-sensitive tabpfn dependency; a different paradigm from the trained-detector template, and the headline is a low-data/few-shot niche (multi-multi acc 0.8765 / AUROC 0.9501 @625 samples) rather than full-data SOTA. Verified repo 2026-06-20: weights on GDrive, frozen DINOv3->PCA(500)->TabPFN.]" + }, + { + "title": "Co-Spy: Fusing Semantic and Artifact Cues for Synthetic Image Detection", + "authors": "Cheng et al.", + "venue": "CVPR", + "year": "2025", + "synthscope": "co_spy", + "github": "https://github.com/Megum1/Co-Spy", + "arxiv": "", + "summary": "Adaptive fusion of a SigLIP semantic stream and an SD-VAE artifact-residual stream.", + "result": "Co-Spy-Bench ~96 mAP; SynthScope retrain reaches 96.6 (beats released weights on that bench)." + }, + { + "title": "SAFE: High-Frequency Artifact Detector with Targeted Augmentation", + "authors": "Ouxiang Li et al.", + "venue": "KDD", + "year": "2025", + "synthscope": "safe", + "github": "https://github.com/Ouxiang-Li/SAFE", + "arxiv": "", + "summary": "Lightweight DWT high-frequency artifact detector with a carefully designed augmentation recipe.", + "result": "SynthScope-trained reproduction (served): ForenSynths-8 97.79 / UFD-8 98.44 / GenImage-8 99.51 / DiTFake 99.96 mAP \u2014 beats the official checkpoint on GenImage." + }, + { + "title": "AIDE: ConvNeXt Semantic + Low-Level DCT/SRM Cues", + "authors": "Yan et al.", + "venue": "ICLR", + "year": "2025", + "synthscope": "aide", + "github": "", + "arxiv": "https://arxiv.org/abs/2406.19435", + "summary": "Fuses ConvNeXt semantic features with low-level DCT/SRM frequency cues; native-resolution preprocessing.", + "result": "GenImage 97.12 mAP / AIGCDetect 97.94 mAP (official weights)." + }, + { + "title": "SPAI: Spectral AI-Image Detection (any-resolution)", + "authors": "Karageorgiou et al.", + "venue": "CVPR", + "year": "2025", + "synthscope": "spai", + "github": "https://github.com/mever-team/spai", + "arxiv": "", + "summary": "Spectral Reconstruction Similarity + Spectral Context Attention; detects at native/arbitrary resolution.", + "result": "SynthScope serves the official weights at native resolution (the faithful any-res path): GenImage 94.22 / Synthbuster 96.75 mAP \u2014 reproduces & beats the vendored-official 91.33." + }, + { + "title": "FatFormer: Forgery-Aware Adaptive Transformer", + "authors": "Liu et al.", + "venue": "CVPR", + "year": "2024", + "synthscope": "fat_former", + "github": "https://github.com/Michel-liu/FatFormer", + "arxiv": "", + "summary": "Frozen CLIP + forgery-aware adapters + learnable text-prompt context.", + "result": "ForenSynths-8 99.7 mAP (official weights)." + }, + { + "title": "NPR: Neighboring-Pixel Relationships", + "authors": "Tan et al.", + "venue": "CVPR", + "year": "2024", + "synthscope": "npr", + "github": "https://github.com/chuangchuangtan/NPR-DeepfakeDetection", + "arxiv": "", + "summary": "Small ResNet on neighboring-pixel residuals that expose generator up-sampling artifacts.", + "result": "GANGen 96.49 mAP / DiffusionForensics 99.4 mAP (official weights)." + }, + { + "title": "UnivFD / CLIP: Towards Universal Fake Image Detectors", + "authors": "Ojha et al.", + "venue": "CVPR", + "year": "2023", + "synthscope": "clip", + "github": "https://github.com/WisconsinAIVision/UniversalFakeDetect", + "arxiv": "", + "summary": "Linear probe on frozen CLIP ViT-L/14 features \u2014 a simple, strong cross-generator baseline.", + "result": "UniversalFakeDetect 95.66 mAP; SynthScope-trained 96.44." + }, + { + "title": "MoE-JEPA: Unifying Semantic Priors and High-Frequency Traces (V-JEPA for synthetic image forensics)", + "authors": "ALCOR-Lab-DIAG", + "venue": "ECCV'26 Workshop (AI for Multimedia Forensics)", + "year": "2026", + "github": "https://github.com/ALCOR-Lab-DIAG/MoE-JEPA", + "arxiv": "https://arxiv.org/abs/2609.16778", + "summary": "Frozen V-JEPA 2 ViT-L backbone with a Residual Mixture-of-Experts injected into the last blocks, a Bayar-conv noise branch and Gated-Attention MIL pooling; uses JEPA 'world understanding' as a semantic prior.", + "result": "95.54% acc / 94.21 F1 on SID-Set (300K images), outperforming much larger models (paper). [integration pending (assessed 2026-10-02): the authors' trained weights are NOT linked (only the frozen V-JEPA 2 backbone is downloaded separately) and there is no single-image inference entrypoint; can only be retrained. Re-checked 2026-10-05: unchanged.]" + }, + { + "title": "RAID: Towards Robust AI-Generated Image Detection with Bit-Reversed Images", + "authors": "renxi-seu", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/renxi-seu/RAID", + "arxiv": "https://arxiv.org/abs/2607.28974", + "summary": "Detects via bit-planes: a bit-reversed image, gradient-based patch selection and a small convolutional classifier, with a theoretical analysis; ~100x faster than counterparts.", + "result": "Claims to outperform existing methods on 40+ benchmarks across cross-generator, cross-dataset and zero-shot settings (paper; README gives no numbers). [integration pending (assessed 2026-10-02): the SD-v1.4-trained weights are only shared via Baidu Cloud (not fetchable programmatically), and the README leaves the architecture and patch-score aggregation unspecified; revisit if a direct weight link appears. Re-checked 2026-10-05: unchanged.]" + }, + { + "title": "Fleet: Few Shots Lead Effective AI-generated Image Detection", + "authors": "ICTMCG", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/ICTMCG/Fleet", + "arxiv": "https://arxiv.org/abs/2606.31082", + "summary": "Dynamic few-shot adaptation: 'constrained routing correction' redirects novel AI samples away from non-AI-dominated routes in decoupled subspaces (DINOv3 backbone), so the detector adapts to rapidly-evolving/commercial generators from a handful of labeled examples. Introduces the Treasure benchmark (64 generators, 360k images, 20 closed-source commercial engines).", + "result": "On a modern generator (Doubao Seedream 4.0), 20.4% -> 73.1% with only 10-shot adaptation, where static SOTA collapses (paper). Repo now live with code (re-listed 2026-10-03; it had been pruned 2026-07-04 while an empty placeholder). [integration pending (assessed 2026-10-03): not a single-checkpoint detector \u2014 the Hugging Face repo wangjiaan/Fleet holds NINE per-target-generator few-shot adapted models (ADM, FLUX.2, GPT4O, HunyuanImage-3.0, NextStep, Qwen-Image, StarGAN, wan2.5, ...; 1.3 GB each), the generic stage-1 base is only on Baidu Pan, inference requires few-shot support data per target generator, and the Treasure benchmark is gated. No clean forward->logit that represents the method; revisit if a generic checkpoint appears on HF.]" + } + ], + "video": [ + { + "title": "D3: Disentangled Detail Dynamics (training-free)", + "authors": "Zheng et al.", + "venue": "ICCV", + "year": "2025", + "synthscope": "d3", + "github": "https://github.com/Zig-HS/D3", + "arxiv": "", + "summary": "Training-free: 2nd-order std of frozen per-frame features as the score.", + "result": "GenVideo-Val ~98 mAP (paper). SynthScope faithful protocol (X-CLIP-B/16 + 10% edge-crop): AUROC 95.95 / acc 91.7 (the paper's mAP isn't comparable on SynthScope's 10000:56 real:fake pool \u2014 AUROC is the robust metric)." + }, + { + "title": "ReStraV: Representation-Trajectory Geometry", + "authors": "Intern\u00f2 et al.", + "venue": "2025", + "synthscope": "restrav", + "github": "https://github.com/ChristianInterno/ReStraV", + "arxiv": "", + "summary": "Frozen DINOv2 trajectory-geometry features + a standardized MLP head.", + "result": "GenVidBench Main ~93; SynthScope 92.7 mAP." + }, + { + "title": "STALL: Spatio-Temporal whitened Log-Likelihoods (training-free)", + "authors": "Ben Hayun et al.", + "venue": "CVPR", + "year": "2026", + "synthscope": "stall", + "github": "https://github.com/OmerBenHayun/STALL", + "arxiv": "", + "summary": "Training-free spatial + temporal whitened-Gaussian log-likelihoods vs a real-video calibration.", + "result": "GenVideo-Val ~0.80 AUC (paper); SynthScope AUROC 83.5." + }, + { + "title": "ATSS: Anomalous Temporal Self-Similarity for AI-Generated Video Detection", + "authors": "Wang et al. (hwang-cs-ime)", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/hwang-cs-ime/ATSS", + "arxiv": "https://arxiv.org/abs/2604.04029", + "summary": "Multimodal detector that exploits anomalous temporal self-similarity fingerprints across frames of AI-generated video. Inference code + a pretrained checkpoint (best.pt) are released; the data sampling/extraction pipeline is still being cleaned for release.", + "result": "Claims SOTA in AP/AUC/ACC over GenVideo, EvalCrafter, VideoPhy, and VidProM vs prior methods (e.g. DeMamba). Repo live 2026-06-21 with code+weights; per-generator numbers not yet in the README. [integration pending: the released checkpoint's forward() consumes PRECOMPUTED multimodal temporal-self-similarity tensors (sim_img/sim_txt/sim_cross, BLIP2-based), not raw video \u2014 and the feature-extraction/data-sampling front-end that produces them is 'still being cleaned for release (coming soon)'. SynthScope feeds raw video clips, so there is no faithful way to run ATSS until that pipeline is published; will revisit when released. Re-checked 2026-10-05: unchanged.]" + }, + { + "title": "G2VD: Generalizable AI-Generated Video Detection via Counterfactual Intervention and Causal Disentanglement", + "authors": "Du et al. (dumeng98)", + "venue": "arXiv", + "year": "2026", + "github": "https://github.com/DMOSCAR-98/G2VD", + "arxiv": "https://arxiv.org/abs/2607.04607", + "summary": "Counterfactual-intervention + causal-disentanglement detector: a training-time CFIPipeline synthesizes controlled counterfactual samples (VAEs + frequency/pixel alignment), and a causal-disentanglement classifier with two domain-anchored branches and an HSIC independence constraint isolates the generalizable artifact signal from domain-specific content.", + "result": "GenVidBench cross-domain: >90% accuracy / AUC ~0.95 using only 10% of the training data (paper). Repo live 2026-07-07 (CLIP/XCLIP/DeMamba backbones \u2014 the same family as SynthScope's D3). [integration pending (re-checked 2026-10-05): repo moved from dumeng98/G2VD (now 404) to DMOSCAR-98/G2VD; checkpoints are still NOT released ('not stored in this repository', MODEL_ZOO.md lists no links) and the CFIPipeline VAE-pool is part of the staged pipeline. Revisit when checkpoints appear.]" + }, + { + "title": "RIFT: Mind the Rift - Cross-Scale Coupling Mismatch for AI-Generated Video Detection", + "authors": "Litsay", + "venue": "arXiv (submitted ACM MM'26)", + "year": "2026", + "github": "https://github.com/Litsay/RIFT", + "arxiv": "https://arxiv.org/abs/2609.00742", + "summary": "Treats a cross-scale coupling mismatch between macro temporal dynamics (differential geometry / persistent homology on frozen DINOv2 features) and micro residuals (steganalytic filtering + RAFT motion) as the forensic signal, scored by a coupling-divergence module.", + "result": "Own benchmark: 99.22% acc / 99.33 F1 / 99.96 AUC, 97.87% leave-one-generator-out (paper; no head-to-head against named prior detectors in the abstract, so not directly comparable). [integration pending (assessed 2026-10-02): trained weights are 'released upon acceptance' (not available), and it needs DINOv2 + RAFT features over ~120K training videos. Re-checked 2026-10-05: unchanged.]" + } + ], + "landscape": [ + { + "title": "GenImage: million-scale real-vs-fake benchmark", + "github": "https://github.com/GenImage-Dataset/GenImage", + "summary": "ImageNet-1k reals + ~equal synthetic from diverse generators." + }, + { + "title": "AIGIBench: robustness/generalization benchmark (23 fake subsets)", + "github": "https://github.com/HorizonTEL/AIGIBench", + "summary": "Multi-source generalization, degradation robustness, augmentation sensitivity, preprocessing impact." + }, + { + "title": "AI-GenBench: temporal (GAN\u2192diffusion) generalization benchmark", + "github": "https://github.com/MI-BioLab/AI-GenBench", + "summary": "Incremental, historically-ordered training to test generalization to new generators (Verimedia @ IJCNN 2025)." + }, + { + "title": "GenVidBench: 6M AI-generated video detection benchmark", + "github": "https://arxiv.org/abs/2501.11340", + "summary": "Large-scale benchmark for AI-generated video detection." + }, + { + "title": "DeepfakeBench: comprehensive deepfake-detection benchmark", + "github": "https://github.com/SCLBD/DeepfakeBench", + "summary": "Unified benchmark across many deepfake datasets/detectors." + }, + { + "title": "Awesome-AIGC-Image-Video-Detection (curated list)", + "github": "https://github.com/ant-research/Awesome-AIGC-Image-Video-Detection", + "summary": "Continuously-updated collection of image + video AIGC-detection papers and code." + }, + { + "title": "Awesome-Comprehensive-Deepfake-Detection (curated list)", + "github": "https://github.com/qiqitao77/Awesome-Comprehensive-Deepfake-Detection", + "summary": "Broad survey-style index of deepfake-detection research and repos." + } + ] +} \ No newline at end of file diff --git a/interface/frontend/index.html b/interface/frontend/index.html new file mode 100644 index 0000000000000000000000000000000000000000..7b7acb03683d907582068a76bee1a033185def60 --- /dev/null +++ b/interface/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + SynthScope — AI-Generated Media Detector + + +
+ + + diff --git a/interface/frontend/package.json b/interface/frontend/package.json new file mode 100644 index 0000000000000000000000000000000000000000..4f42e84c5b85f9b9b0ca3032a77e47e0931ab743 --- /dev/null +++ b/interface/frontend/package.json @@ -0,0 +1,20 @@ +{ + "name": "synthscope-interface", + "private": true, + "version": "1.0.0", + "type": "module", + "description": "Web UI for the SynthScope AI-generated media detectors", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@vitejs/plugin-react": "^4.2.1", + "vite": "^5.2.0" + } +} diff --git a/interface/frontend/src/App.jsx b/interface/frontend/src/App.jsx new file mode 100644 index 0000000000000000000000000000000000000000..933dcce054d0cb42d8259098b59f365abdbf9453 --- /dev/null +++ b/interface/frontend/src/App.jsx @@ -0,0 +1,649 @@ +import React, { useEffect, useRef, useState } from "react"; +import { + fetchModels, predict, fetchDatasets, sampleDataset, datasetMediaUrl, + predictDataset, scoreBatch, verdictAt, autoCalibrate, targetFprThreshold, batchStats, + sotaGenerators, sotaRealImage, sotaCompare, fetchLeaderboard, fetchGeneratorBenchmarks, + fetchTrending, +} from "./api.js"; + +// Which weights back the default inference checkpoint for each detector (shown in the UI). +const WEIGHTS_LABEL = { + official: "official weights", + synthscope_trained: "SynthScope-trained", + training_free: "training-free", +}; +const wlabel = (m) => WEIGHTS_LABEL[m && m.weights] || "SynthScope-trained"; + +const REPRO_LEGEND = ( +
+ reproduced = SynthScope-trained checkpoint matches the paper · official = official weights loaded + into the SynthScope class match (SynthScope-trained shown alongside where trained from scratch) · below = gap + remains · reference = paper reports no comparable number · missing = not yet evaluated. +
+); + +function TrendingView() { + const [data, setData] = useState(null); + const [err, setErr] = useState(""); + useEffect(() => { fetchTrending().then(setData).catch((e) => setErr(e.message)); }, []); + if (err) return
{err}
; + if (!data) return

Loading…

; + const card = (e, i) => { + const link = e.github || e.arxiv || ""; + const venue = [e.authors, e.venue, e.year].filter(Boolean).join(" · "); + return ( +
+
+ {link ? {e.title} : {e.title}} + {e.synthscope && in SynthScope: {e.synthscope}} +
+ {venue &&
{venue}
} + {e.summary &&
{e.summary}
} + {e.result &&
Results: {e.result}
} +
+ {e.github && code ↗} + {e.arxiv && arXiv ↗} +
+
+ ); + }; + return ( +
+

📈 What's Trending — AIGC detection

+

{data.criteria}{data.updated ? ` Last updated: ${data.updated}.` : ""}

+

Image detectors

+
{(data.image || []).map(card)}
+

Video detectors

+
{(data.video || []).map(card)}
+

Benchmarks & curated lists

+
    + {(data.landscape || []).map((e, i) => ( +
  • {e.title} — {e.summary}
    • + ))} +
+
+ ); +} + +function LeaderboardView() { + const [groups, setGroups] = useState([]); + const [sel, setSel] = useState(""); + const [err, setErr] = useState(""); + useEffect(() => { + fetchLeaderboard().then((d) => setGroups(d.groups || [])).catch((e) => setErr(e.message)); + }, []); + const shown = sel ? groups.filter((g) => g.benchmark === sel) : groups; + return ( +
+
+

Every benchmark each paper reports, with the paper number, our SynthScope-trained + checkpoint, and official weights loaded into the SynthScope class. Grouped by benchmark so methods that + share one can be compared head-to-head (sorted best-first).

+ +
+ {err &&
{err}
} + {shown.map((g) => ( +
+

{g.benchmark} {g.n_methods} method{g.n_methods > 1 ? "s" : ""}

+
+ + + + {g.rows.map((r, i) => ( + + + + + + ))} + +
MethodPaperSynthScope (trained)Official wtsStatus
{r.method}
{r.benchmark}
{r.paper}{r.synthscope || "—"}{r.official || "—"}{r.status}
+
+
+ ))} + {REPRO_LEGEND} +
+ ); +} + +function TopGeneratorsView() { + const [bm, setBm] = useState([]); + const [err, setErr] = useState(""); + useEffect(() => { + fetchGeneratorBenchmarks().then((d) => setBm(d.benchmarks || [])).catch((e) => setErr(e.message)); + }, []); + const medal = (i) => (i === 0 ? "🥇" : i === 1 ? "🥈" : i === 2 ? "🥉" : i + 1); + return ( +
+

How our detectors hold up against today's SOTA image generators. Each: a balanced + real-vs-generated set scored by every SynthScope detector (promoted checkpoints), ranked by AUROC. None of the + detectors were trained on these generators — cross-generator generalization.

+ {err &&
{err}
} + {!bm.length && !err &&
No generator benchmarks yet.
} + {bm.map((b) => ( +
+

vs {b.generator} {b.n_real}+{b.n_fake} imgs · {b.settings}

+
{b.note}
+
+ + + + {b.rows.map((r, i) => ( + + + + + + ))} + +
#DetectorAUROCAPacc@.5fake-detect %real-FP %
{medal(i)}{r.name || r.detector}{r.auroc}{r.ap}{r.acc}{r.fake_detect_rate}{r.real_false_pos}
+
+
+ ))} +
+ ); +} + +function b64ToFile(b64, name) { + const bin = atob(b64), arr = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) arr[i] = bin.charCodeAt(i); + return new File([arr], name, { type: "image/png" }); +} + +// SOTA generator vs detector comparison: pick/upload a real image -> a current SOTA generator makes a +// "conceptually similar" image -> the selected SynthScope detectors score it (best/promoted checkpoints). +function SotaView({ models }) { + const [gens, setGens] = useState([]); + const [gen, setGen] = useState(""); + const [prompt, setPrompt] = useState("make another image conceptually similar to this"); + const [realB64, setRealB64] = useState(""); + const [realSrc, setRealSrc] = useState(null); + const [picked, setPicked] = useState({}); + const [busy, setBusy] = useState(false); + const [res, setRes] = useState(null); + const [err, setErr] = useState(""); + const imgDetectors = (models || []).filter((m) => m.modality === "image" && m.available); + + useEffect(() => { + sotaGenerators().then((d) => { + setGens(d.generators || []); + const a = (d.generators || []).find((g) => g.available); + if (a) setGen(a.key); + if (d.default_prompt) setPrompt(d.default_prompt); + }).catch((e) => setErr(e.message)); + }, []); + + async function rollImage() { + setErr(""); setRes(null); + try { const d = await sotaRealImage(); setRealB64(d.image_b64); setRealSrc(d.source); } + catch (e) { setErr(e.message); } + } + function onUpload(f) { + if (!f) return; + const rd = new FileReader(); + rd.onload = () => { setRealB64(rd.result.split(",")[1]); setRealSrc(["upload", f.name]); setRes(null); }; + rd.readAsDataURL(f); + } + async function run() { + setErr(""); setBusy(true); setRes(null); + try { + const dets = Object.keys(picked).filter((k) => picked[k]); + if (!realB64) throw new Error("Pick or upload a real image first."); + if (!gen) throw new Error("Select a generator."); + if (!dets.length) throw new Error("Select at least one detector."); + const d = await sotaCompare(b64ToFile(realB64, "real.png"), gen, dets, prompt); + setRes(d); + } catch (e) { setErr(e.message); } finally { setBusy(false); } + } + + return ( +
+

Send a real image from our datasets to a current SOTA generator, then see whether + the SynthScope detectors catch the freshly-generated image. Generators need an API key + (greyed-out = key not set on the server).

+
+
+ + +
+ + +
+
Detectors: + {imgDetectors.map((m) => ( + + ))} +
+ + {err &&
{err}
} +
+ {realB64 &&
real +
Real {realSrc && "· " + realSrc.join("/")}
} + {res && res.generated_b64 &&
generated +
Generated · {res.generator}{res.prompt && <>
prompt: {res.prompt}}
} +
+ {res && ( + + + + {res.results.map((r, i) => ( + + + + + + + ))} + +
DetectorP(fake) realP(fake) generatedCaught?
{r.name || r.detector}{r.error ? "—" : r.real_fake_prob}{r.error ? "—" : r.gen_fake_prob} + {r.error ? r.error : (r.caught ? "✓ fake" : "✗ missed")}
+ )} +
+ ); +} + +const isVideoName = (name = "", type = "") => + type.startsWith("video") || /\.(mp4|mov|avi|mkv|webm|m4v)$/i.test(name); +const optLabel = (m) => `${m.name} — ${m.paper} · ${wlabel(m)}${m.available ? "" : " (no checkpoint)"}`; + +function ModelSelect({ models, model, setModel, onChange }) { + const img = models.filter((m) => m.modality === "image"); + const vid = models.filter((m) => m.modality === "video"); + return ( + + ); +} + +function ReproTable({ m }) { + const rep = m && m.repro; + const full = m && m.repro_full; + if (!rep && !(full && full.length)) return null; + return ( +
+ {rep && ( + + + + + +
Inference weights{wlabel(m)}
Headline{rep.benchmark}
Paper{rep.paper}
Reproduced{rep.ours}
+ )} + {full && full.length > 0 && ( +
+ All {full.length} reported benchmarks — paper vs SynthScope vs official +
+ + + + {full.map((r, i) => ( + + + + + + + + ))} + +
BenchmarkPaperSynthScope (trained)Official wtsStatus
{r.benchmark}{r.paper}{r.ours_synthscope || "—"}{r.ours_official || "—"}{r.status}
+
+
+ reproduced = SynthScope-trained checkpoint matches the paper · official = official weights + loaded into the SynthScope class match (SynthScope-trained number shown where we trained from scratch) · + below = gap remains · reference = paper reports no comparable number · missing = not yet evaluated. +
+
+ )} +
+ ); +} + +function ThresholdSlider({ threshold, setThreshold, children }) { + return ( +
+
Decision thresholdP(fake) ≥ {threshold.toFixed(2)} → fake
+ setThreshold(parseFloat(e.target.value))} /> + {children} +
+ ); +} + +export default function App() { + const [models, setModels] = useState([]); + const [device, setDevice] = useState(""); + const [model, setModel] = useState(""); + const [tab, setTab] = useState("detect"); + const [threshold, setThreshold] = useState(0.5); + const [error, setError] = useState(""); + + useEffect(() => { + fetchModels().then((d) => { + setModels(d.models || []); setDevice(d.device || ""); + const first = (d.models || []).find((m) => m.available); + if (first) setModel(first.key); + }).catch((e) => setError("Could not reach backend: " + e.message)); + }, []); + + const shared = { models, model, setModel, threshold, setThreshold }; + return ( +
+
+

SynthScope — AI-Generated Media Detector

+

Detect AI-generated media, or explore the datasets each detector was trained / tested on. + {device && device: {device}}

+
+
+ + + + + + +
+ {error &&
{error}
} + {tab === "detect" && } + {tab === "explore" && } + {tab === "sota" && } + {tab === "leaderboard" && } + {tab === "topgen" && } + {tab === "trending" && } +
+ ); +} + +function DetectView({ models, model, setModel, threshold, setThreshold }) { + const [file, setFile] = useState(null); + const [previewUrl, setPreviewUrl] = useState(""); + const [isVideo, setIsVideo] = useState(false); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(""); + const [drag, setDrag] = useState(false); + const inputRef = useRef(); + const selected = models.find((m) => m.key === model); + + function onPick(f) { + if (!f) return; + setFile(f); setResult(null); setError(""); + setIsVideo(isVideoName(f.name, f.type)); + if (previewUrl) URL.revokeObjectURL(previewUrl); + setPreviewUrl(URL.createObjectURL(f)); + } + async function run() { + if (!file || !model) return; + setBusy(true); setResult(null); setError(""); + try { setResult(await predict(file, model)); } + catch (e) { setError(e.message); } + finally { setBusy(false); } + } + + return ( +
+
+

Input

+ + setResult(null)} /> + {selected && ( +
{selected.notes} + {selected.modality === "image" ? " · On a video, runs frame-by-frame and averages." : " · Native video model."}
+ )} + {selected && } + +
inputRef.current.click()} + onDragOver={(e) => { e.preventDefault(); setDrag(true); }} + onDragLeave={() => setDrag(false)} + onDrop={(e) => { e.preventDefault(); setDrag(false); onPick(e.dataTransfer.files[0]); }}> + {file ? {file.name} ({(file.size / 1048576).toFixed(2)} MB) + : "Click or drop an image / video here"} +
+ onPick(e.target.files[0])} /> + {previewUrl &&
{isVideo ?
} + + {error &&
{error}
} +
+ +
+

Result

+ {!result && !busy &&
No result yet.
} + {busy &&
Running {selected ? selected.name : ""}… first run loads the model and may take a while.
} + {result && } + {result && } +
+
+ ); +} + +function ExploreView({ models, model, setModel, threshold, setThreshold }) { + const [datasets, setDatasets] = useState([]); + const [trainMap, setTrainMap] = useState({}); + const [dataset, setDataset] = useState(""); + const [split, setSplit] = useState(""); + const [seed, setSeed] = useState(1); + const [items, setItems] = useState([]); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [sel, setSel] = useState(null); + const [preds, setPreds] = useState({}); + const [scoring, setScoring] = useState(false); + const [fpr, setFpr] = useState(5); + const batchN = 24; + + const curDs = datasets.find((d) => d.key === dataset); + const trainDsKey = (trainMap[model] || {}).dataset; + + useEffect(() => { + fetchDatasets().then((d) => { + setDatasets(d.datasets || []); setTrainMap(d.model_train || {}); + const first = (d.datasets || [])[0]; + if (first) { setDataset(first.key); setSplit(first.splits[first.splits.length - 1] || "."); } + }).catch((e) => setError("Could not load datasets: " + e.message)); + }, []); + + useEffect(() => { + if (!curDs) return; + const s = curDs.splits.includes(split) ? split : curDs.splits[curDs.splits.length - 1] || "."; + if (s !== split) setSplit(s); + loadBatch(dataset, s, seed); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dataset]); + + async function loadBatch(dsKey, sp, sd) { + if (!dsKey) return; + setBusy(true); setError(""); setItems([]); setSel(null); setPreds({}); + try { setItems((await sampleDataset(dsKey, sp, batchN, sd)).items || []); } + catch (e) { setError(e.message); } + finally { setBusy(false); } + } + function newBatch() { const sd = seed + 1; setSeed(sd); loadBatch(dataset, split, sd); } + + async function pickItem(it) { + setSel(it.rel); + if (!model || preds[it.rel]) return; + try { setPreds((p) => ({ ...p, [it.rel]: await predictDataset(dataset, it.rel, model) })); } + catch (e) { setPreds((p) => ({ ...p, [it.rel]: { error: e.message } })); } + } + + async function scoreAll() { + if (!model || !items.length) return []; + setScoring(true); setError(""); + try { + const d = await scoreBatch(dataset, model, items.map((i) => i.rel)); + const m = {}; for (const it of d.items) m[it.rel] = it; + setPreds((p) => ({ ...p, ...m })); + return d.items.filter((x) => x.ground_truth && typeof x.fake_probability === "number") + .map((x) => ({ p: x.fake_probability, gt: x.ground_truth })); + } catch (e) { setError(e.message); return []; } + finally { setScoring(false); } + } + + const scored = items.map((i) => preds[i.rel]) + .filter((x) => x && typeof x.fake_probability === "number" && x.ground_truth) + .map((x) => ({ p: x.fake_probability, gt: x.ground_truth })); + + async function calibrate() { const s = scored.length ? scored : await scoreAll(); if (s.length) setThreshold(autoCalibrate(s)); } + async function applyFpr() { + const s = scored.length ? scored : await scoreAll(); + const t = targetFprThreshold(s, fpr / 100); + if (t != null) setThreshold(t); else setError("Need scored real videos for target-FPR."); + } + + const selItem = items.find((i) => i.rel === sel); + const selRes = sel ? preds[sel] : null; + const stats = batchStats(scored, threshold); + const mediaUrl = (it, thumb) => datasetMediaUrl(dataset, it.rel, thumb); + + return ( +
+
+

Datasets

+ + + {trainDsKey && ( +
Trained on {(datasets.find((d) => d.key === trainDsKey) || {}).name || trainDsKey} + {(trainMap[model] || {}).official && official wts}
+ )} + m.key === model)} /> +
+
+
+
+
+
+
+ + +
+ + + + setFpr(parseFloat(e.target.value) || 0)} />% +
+ {stats && ( +
batch ({stats.n} scored) @{threshold.toFixed(2)}: acc {(stats.acc * 100).toFixed(0)}% + {stats.racc != null ? ` · real ${(stats.racc * 100).toFixed(0)}%` : ""} + {stats.facc != null ? ` · fake ${(stats.facc * 100).toFixed(0)}%` : ""}
+ )} + + + {error &&
{error}
} + {busy &&
Sampling…
} +
+ {items.map((it) => { + const pr = preds[it.rel]; + const has = pr && typeof pr.fake_probability === "number"; + const v = has ? verdictAt(pr.fake_probability, threshold) : null; + const ok = has && v === it.label; + return ( +
pickItem(it)} title={it.name}> + + {it.label} + {it.modality === "video" && ▶} + {has && {(pr.fake_probability * 100).toFixed(0)}% {ok ? "✓" : "✗"}} +
+ ); + })} +
+
+ +
+

Model output

+ {!selItem &&
Click a sample to run {(models.find((m) => m.key === model) || {}).name || "the model"} on it. Use "Score batch" + Auto-calibrate / Target FPR to set the threshold.
} + {selItem && ( +
+
+ {selItem.modality === "video" ?
+
{selItem.name} · ground truth: {selItem.label}
+ {selRes && selRes.error &&
{selRes.error}
} + {selRes && !selRes.error && typeof selRes.fake_probability === "number" && } +
+ )} +
+
+ ); +} + +function Result({ r, threshold, gt }) { + const fakePct = r.fake_probability * 100, realPct = (1 - r.fake_probability) * 100; + const verdict = verdictAt(r.fake_probability, threshold); + const correct = gt ? verdict === gt : null; + return ( +
+
{verdict === "fake" ? "AI-GENERATED" : "REAL"}
+
+ P(fake) {fakePct.toFixed(1)}% vs threshold {threshold.toFixed(2)} + {correct != null && <> · {correct ? "correct" : "wrong"}} +
+
+ {fakePct >= 12 ? `fake ${fakePct.toFixed(0)}%` : ""} + {realPct >= 12 ? `real ${realPct.toFixed(0)}%` : ""} + +
+
+ + {r.model_name && } + {r.media_type && } + {r.elapsed_ms && } +
+ {r.per_frame && r.per_frame.length > 1 && ( +
+
Per-frame P(fake) — {r.per_frame.length} frames sampled (dashed line = 50% threshold):
+
+ + {r.per_frame.map((v, i) => ( + = 0.5 ? "var(--fake)" : "var(--accent)" }} /> + ))} +
+
+ min {(Math.min(...r.per_frame) * 100).toFixed(0)}% + mean {(r.per_frame.reduce((a, b) => a + b, 0) / r.per_frame.length * 100).toFixed(0)}% + max {(Math.max(...r.per_frame) * 100).toFixed(0)}% +
+
+ )} + {r.interpret_map && ( +
+
Interpretability — {({safe_hf: "DWT high-frequency band (SAFE)", npr_residual: "neighboring-pixel residual (NPR)", pgc_peak: "peak-guided residual fake-evidence (PGC)", aide_dct: "native-res DCT patch-energy (AIDE)", spai_patch: "per-patch spectral fake-evidence (SPAI)"})[r.interpret_kind] || "artifact map"} · warmer = stronger artifact signal
+
+ {r.interpret_overlay && ( +
overlay on image
overlay on image
+ )} +
heatmap
heatmap
+
+
+ )} +
+ ); +} + +const Row = ({ k, v }) => (
{k}{v}
); diff --git a/interface/frontend/src/api.js b/interface/frontend/src/api.js new file mode 100644 index 0000000000000000000000000000000000000000..6aa878637e87ee9aca691b1596f288f09b8f55c2 --- /dev/null +++ b/interface/frontend/src/api.js @@ -0,0 +1,131 @@ +// API client for the SynthScope backend. In dev, Vite proxies /api to the FastAPI server. +const BASE = import.meta.env.VITE_API_BASE || ""; + +export async function fetchModels() { + const r = await fetch(`${BASE}/api/models`); + if (!r.ok) throw new Error(`models: HTTP ${r.status}`); + return r.json(); +} + +export async function predict(file, model) { + const fd = new FormData(); + fd.append("file", file); + fd.append("model", model); + const r = await fetch(`${BASE}/api/predict`, { method: "POST", body: fd }); + const d = await r.json(); + if (!r.ok) throw new Error(d.detail || `HTTP ${r.status}`); + return d; +} + +// --- dataset explorer -------------------------------------------------------- +export async function fetchDatasets() { + const r = await fetch(`${BASE}/api/datasets`); + if (!r.ok) throw new Error(`datasets: HTTP ${r.status}`); + return r.json(); +} + +export async function sampleDataset(key, split, n, seed) { + const u = `${BASE}/api/datasets/${key}/sample?split=${encodeURIComponent(split)}&n=${n}&seed=${seed}`; + const r = await fetch(u); + const d = await r.json(); + if (!r.ok) throw new Error(d.detail || `HTTP ${r.status}`); + return d; +} + +export function datasetMediaUrl(key, rel, thumb) { + return `${BASE}/api/datasets/${key}/media?rel=${encodeURIComponent(rel)}${thumb ? "&thumb=1" : ""}`; +} + +export async function predictDataset(dataset, rel, model) { + const fd = new FormData(); + fd.append("dataset", dataset); + fd.append("rel", rel); + fd.append("model", model); + const r = await fetch(`${BASE}/api/predict_dataset`, { method: "POST", body: fd }); + const d = await r.json(); + if (!r.ok) throw new Error(d.detail || `HTTP ${r.status}`); + return d; +} + +export async function scoreBatch(dataset, model, rels) { + const r = await fetch(`${BASE}/api/score_batch`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ dataset, model, rels }), + }); + const d = await r.json(); + if (!r.ok) throw new Error(d.detail || `HTTP ${r.status}`); + return d; +} + +// --- threshold calibration helpers (shared by Detect + Explorer) --- +export const verdictAt = (p, thr) => (p >= thr ? "fake" : "real"); + +export function autoCalibrate(scored) { + let best = 0.5, bestAcc = -1; + for (let t = 0; t <= 1.0001; t += 0.01) { + const reals = scored.filter((s) => s.gt === "real"), fakes = scored.filter((s) => s.gt === "fake"); + const tnr = reals.length ? reals.filter((s) => s.p < t).length / reals.length : 0; + const tpr = fakes.length ? fakes.filter((s) => s.p >= t).length / fakes.length : 0; + const acc = (tpr + tnr) / 2; + if (acc > bestAcc) { bestAcc = acc; best = t; } + } + return Math.round(best * 100) / 100; +} + +export function targetFprThreshold(scored, fpr) { + const reals = scored.filter((s) => s.gt === "real").map((s) => s.p).sort((a, b) => a - b); + if (!reals.length) return null; + const idx = Math.min(reals.length - 1, Math.max(0, Math.ceil((1 - fpr) * reals.length) - 1)); + return Math.round(reals[idx] * 100) / 100; +} + +export function batchStats(scored, thr) { + if (!scored.length) return null; + const reals = scored.filter((s) => s.gt === "real"), fakes = scored.filter((s) => s.gt === "fake"); + return { + n: scored.length, + acc: scored.filter((s) => verdictAt(s.p, thr) === s.gt).length / scored.length, + racc: reals.length ? reals.filter((s) => s.p < thr).length / reals.length : null, + facc: fakes.length ? fakes.filter((s) => s.p >= thr).length / fakes.length : null, + }; +} + +// --- SOTA generator vs detector comparison ---------------------------------- +export async function sotaGenerators() { + const r = await fetch(`${BASE}/api/sota/generators`); + if (!r.ok) throw new Error(`generators: HTTP ${r.status}`); + return r.json(); +} +export async function sotaRealImage() { + const r = await fetch(`${BASE}/api/sota/real_image`); + const d = await r.json(); + if (!r.ok) throw new Error(d.detail || `HTTP ${r.status}`); + return d; +} +export async function fetchLeaderboard() { + const r = await fetch(`${BASE}/api/leaderboard`); + if (!r.ok) throw new Error(`leaderboard: HTTP ${r.status}`); + return r.json(); +} +export async function fetchGeneratorBenchmarks() { + const r = await fetch(`${BASE}/api/generator_benchmarks`); + if (!r.ok) throw new Error(`generator benchmarks: HTTP ${r.status}`); + return r.json(); +} +export async function fetchTrending() { + const r = await fetch(`${BASE}/api/trending`); + if (!r.ok) throw new Error(`trending: HTTP ${r.status}`); + return r.json(); +} +export async function sotaCompare(file, generator, detectors, prompt) { + const fd = new FormData(); + fd.append("file", file); + fd.append("generator", generator); + fd.append("detectors", detectors.join(",")); + fd.append("prompt", prompt); + const r = await fetch(`${BASE}/api/sota/compare`, { method: "POST", body: fd }); + const d = await r.json(); + if (!r.ok) throw new Error(d.detail || `HTTP ${r.status}`); + return d; +} diff --git a/interface/frontend/src/main.jsx b/interface/frontend/src/main.jsx new file mode 100644 index 0000000000000000000000000000000000000000..4ecd9cfb14e62fca48ebd0ecd6d9784abf85a620 --- /dev/null +++ b/interface/frontend/src/main.jsx @@ -0,0 +1,10 @@ +import React from "react"; +import { createRoot } from "react-dom/client"; +import App from "./App.jsx"; +import "./styles.css"; + +createRoot(document.getElementById("root")).render( + + + +); diff --git a/interface/frontend/src/styles.css b/interface/frontend/src/styles.css new file mode 100644 index 0000000000000000000000000000000000000000..1b8fba1c4f58c960bb37c5c670ba7e1bdb1594bb --- /dev/null +++ b/interface/frontend/src/styles.css @@ -0,0 +1,298 @@ +/* ============================================================================ + SynthScope web interface — visual theme + Modern dark "glass dashboard": layered navy gradient, frosted panels, an + indigo→cyan accent, soft shadows, and smooth motion. Class names are kept + stable so App.jsx is untouched. + ========================================================================== */ +:root { + --bg: #070b16; + --bg2: #0c1326; + --panel: rgba(22, 29, 51, 0.72); + --panel-solid: #161d33; + --panel2: rgba(31, 41, 71, 0.55); + --line: rgba(120, 140, 190, 0.16); + --line-strong: rgba(120, 140, 190, 0.30); + --text: #eef2fb; + --muted: #9aa7c7; + --muted2: #6b7796; + --accent: #7c8cff; + --accent2: #46d6ff; + --accent-grad: linear-gradient(135deg, #7c8cff 0%, #46d6ff 100%); + --real: #34d399; + --fake: #ff6b7d; + --radius: 16px; + --radius-sm: 10px; + --shadow: 0 10px 30px -12px rgba(0, 0, 0, 0.65); + --shadow-lg: 0 24px 60px -20px rgba(0, 0, 0, 0.75); +} + +* { box-sizing: border-box; } + +body { + margin: 0; + color: var(--text); + font-family: "Inter", system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + -webkit-font-smoothing: antialiased; + background: + radial-gradient(1100px 600px at 85% -10%, rgba(70, 214, 255, 0.10), transparent 60%), + radial-gradient(1000px 700px at -5% 0%, rgba(124, 140, 255, 0.14), transparent 55%), + linear-gradient(180deg, var(--bg2) 0%, var(--bg) 100%); + background-attachment: fixed; + min-height: 100vh; +} + +.wrap { max-width: 1100px; margin: 0 auto; padding: 40px 22px 80px; } + +/* ---- header ---------------------------------------------------------------*/ +header { margin-bottom: 6px; } +header h1 { + margin: 0 0 6px; font-size: 34px; font-weight: 800; letter-spacing: -0.02em; + background: var(--accent-grad); -webkit-background-clip: text; + background-clip: text; -webkit-text-fill-color: transparent; +} +header p { margin: 0; color: var(--muted); font-size: 15px; max-width: 720px; line-height: 1.5; } + +/* ---- layout ---------------------------------------------------------------*/ +.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 22px; margin-top: 26px; } +@media (max-width: 860px) { .grid { grid-template-columns: 1fr; } } + +.card { + background: var(--panel); + border: 1px solid var(--line); + border-radius: var(--radius); + padding: 22px; + box-shadow: var(--shadow); + backdrop-filter: blur(14px) saturate(120%); + -webkit-backdrop-filter: blur(14px) saturate(120%); +} +.card h2 { + margin: 0 0 14px; font-size: 12px; color: var(--muted); + text-transform: uppercase; letter-spacing: 0.12em; font-weight: 700; +} + +label { display: block; font-size: 13px; color: var(--muted); margin: 14px 0 7px; font-weight: 500; } + +select, input[type=text], input[type=number] { + width: 100%; padding: 11px 12px; border-radius: var(--radius-sm); + background: var(--panel2); color: var(--text); border: 1px solid var(--line-strong); + font-size: 14px; transition: border-color .15s, box-shadow .15s; outline: none; +} +select:focus, input[type=text]:focus, input[type=number]:focus { + border-color: var(--accent); box-shadow: 0 0 0 3px rgba(124, 140, 255, 0.22); +} +option { background: var(--panel-solid); } +option:disabled { color: #5a647e; } + +/* ---- upload / drop --------------------------------------------------------*/ +.drop { + border: 1.5px dashed var(--line-strong); border-radius: var(--radius); + padding: 30px; text-align: center; cursor: pointer; color: var(--muted); + transition: border-color .18s, background .18s, color .18s, transform .18s; +} +.drop:hover { border-color: var(--accent); color: var(--text); } +.drop.drag { border-color: var(--accent2); background: rgba(70, 214, 255, 0.08); color: var(--text); transform: scale(1.01); } +.preview { + margin-top: 16px; border-radius: var(--radius-sm); overflow: hidden; background: #000; + max-height: 320px; display: flex; justify-content: center; border: 1px solid var(--line); +} +.preview img, .preview video { max-width: 100%; max-height: 320px; display: block; } + +/* ---- buttons --------------------------------------------------------------*/ +button.go, button.sm, button.primary { + border: 0; border-radius: var(--radius-sm); background: var(--accent-grad); + color: #06122b; font-weight: 700; cursor: pointer; + box-shadow: 0 8px 20px -8px rgba(124, 140, 255, 0.6); + transition: transform .12s ease, box-shadow .12s ease, opacity .12s; +} +button.go { margin-top: 18px; width: 100%; padding: 13px; font-size: 15px; } +button.sm { padding: 11px 16px; white-space: nowrap; } +button.primary { padding: 9px 18px; } +button.go:hover:not(:disabled), button.sm:hover:not(:disabled), button.primary:hover:not(:disabled) { + transform: translateY(-1px); box-shadow: 0 12px 26px -8px rgba(70, 214, 255, 0.55); +} +button.go:active, button.sm:active { transform: translateY(0); } +button.go:disabled, button.sm:disabled, button.primary:disabled { opacity: .4; cursor: not-allowed; box-shadow: none; } + +.meta { font-size: 12px; color: var(--muted); margin-top: 12px; line-height: 1.6; } + +/* ---- verdict / probability ------------------------------------------------*/ +.verdict { font-size: 38px; font-weight: 800; text-align: center; margin: 6px 0 2px; letter-spacing: -0.02em; } +.verdict.real { color: var(--real); text-shadow: 0 0 28px rgba(52, 211, 153, 0.35); } +.verdict.fake { color: var(--fake); text-shadow: 0 0 28px rgba(255, 107, 125, 0.35); } + +.bar { + position: relative; height: 28px; border-radius: 9px; overflow: hidden; display: flex; + background: var(--panel2); border: 1px solid var(--line); margin: 16px 0 6px; +} +.bar .fake { background: linear-gradient(90deg, #ff6b7d, #ff8a5d); } +.bar .real { background: linear-gradient(90deg, #34d399, #46d6ff); } +.bar span { + display: flex; align-items: center; justify-content: center; + font-size: 12px; font-weight: 700; color: #07101f; white-space: nowrap; transition: flex-basis .4s ease; +} +.bar .thrline { position: absolute; top: -3px; bottom: -3px; width: 2px; background: #fff; box-shadow: 0 0 6px rgba(255,255,255,.6); } + +.row { + display: flex; justify-content: space-between; font-size: 13px; + padding: 8px 0; border-bottom: 1px solid var(--line); color: var(--muted); +} +.row:last-child { border-bottom: 0; } +.row b { color: var(--text); } + +/* ---- per-frame sparkline --------------------------------------------------*/ +.frames { margin-top: 14px; } +.spark { + position: relative; display: flex; gap: 2px; align-items: flex-end; height: 64px; margin-top: 8px; + padding: 4px 4px 0; background: var(--panel2); border: 1px solid var(--line); border-radius: var(--radius-sm); +} +.spark i { flex: 1; background: var(--accent-grad); border-radius: 3px 3px 0 0; min-height: 3px; opacity: .9; } +.spark .thr50 { position: absolute; left: 6px; right: 6px; top: 50%; height: 0; border-top: 1px dashed #4a5572; pointer-events: none; } +.frame-stats { display: flex; justify-content: space-between; font-size: 11px; color: var(--muted); margin-top: 5px; } +.frame-stats b { color: var(--text); } + +.err { color: var(--fake); font-size: 13px; margin-top: 12px; white-space: pre-wrap; } +.pill { + display: inline-block; font-size: 11px; padding: 2px 9px; border-radius: 999px; + background: var(--panel2); border: 1px solid var(--line-strong); color: var(--muted); margin-left: 8px; +} +.empty { color: var(--muted2); text-align: center; padding: 48px 0; font-size: 14px; } + +/* ---- tabs -----------------------------------------------------------------*/ +.tabs { display: flex; gap: 4px; margin-top: 26px; border-bottom: 1px solid var(--line); flex-wrap: wrap; } +.tab { + background: transparent; border: 0; color: var(--muted); padding: 11px 18px; + cursor: pointer; font-size: 14px; font-weight: 600; border-bottom: 2px solid transparent; + transition: color .15s, border-color .15s; +} +.tab:hover { color: var(--text); } +.tab.active { color: var(--text); border-bottom-color: transparent; + background: var(--accent-grad); -webkit-background-clip: text; background-clip: text; -webkit-text-fill-color: transparent; + border-bottom: 2px solid var(--accent); } + +/* ---- data explorer --------------------------------------------------------*/ +.toolbar { display: flex; gap: 12px; align-items: flex-end; flex-wrap: wrap; } +.toolbar > div { flex: 1; min-width: 150px; } +.badge { + display: inline-block; font-size: 10px; padding: 1px 8px; border-radius: 7px; + background: var(--accent-grad); color: #06122b; font-weight: 700; margin-left: 6px; vertical-align: middle; +} +.thumbs { display: grid; grid-template-columns: repeat(auto-fill, minmax(112px, 1fr)); gap: 11px; margin-top: 18px; } +.thumb { + position: relative; border-radius: var(--radius-sm); overflow: hidden; cursor: pointer; + border: 2px solid transparent; background: #000; aspect-ratio: 1 / 1; transition: transform .15s, border-color .15s; +} +.thumb:hover { transform: translateY(-2px); } +.thumb.sel { border-color: var(--accent); box-shadow: 0 0 0 3px rgba(124, 140, 255, 0.25); } +.thumb img { width: 100%; height: 100%; object-fit: cover; display: block; } +.thumb .tag { position: absolute; top: 5px; left: 5px; font-size: 10px; padding: 1px 7px; border-radius: 6px; font-weight: 700; letter-spacing: .03em; } +.thumb .tag.real { background: var(--real); color: #06210f; } +.thumb .tag.fake { background: var(--fake); color: #2a0707; } +.thumb .pred { position: absolute; bottom: 0; left: 0; right: 0; font-size: 10px; text-align: center; padding: 2px 0; font-weight: 700; } +.thumb .pred.ok { background: rgba(52, 211, 153, .92); color: #06210f; } +.thumb .pred.no { background: rgba(255, 107, 125, .92); color: #fff; } +.thumb .play { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; font-size: 24px; color: rgba(255,255,255,.85); pointer-events: none; text-shadow: 0 2px 8px rgba(0,0,0,.6); } + +/* ---- threshold + calibration ----------------------------------------------*/ +.thr { margin-top: 16px; padding: 14px; border: 1px solid var(--line); border-radius: var(--radius-sm); background: var(--panel2); } +.thr input[type=range] { width: 100%; accent-color: var(--accent); } +.thr .lab { font-size: 12px; color: var(--muted); display: flex; justify-content: space-between; } +.thr .lab b { color: var(--text); } +.calib { display: flex; gap: 8px; flex-wrap: wrap; align-items: center; margin-top: 12px; } +.calib button { padding: 8px 12px; border: 1px solid var(--line-strong); border-radius: 8px; background: var(--panel2); color: var(--text); font-size: 12px; font-weight: 600; cursor: pointer; transition: border-color .15s; } +.calib button:hover:not(:disabled) { border-color: var(--accent); } +.calib button:disabled { opacity: .45; cursor: not-allowed; } +.calib input[type=number] { width: 64px; padding: 6px; } +.stat { font-size: 12px; color: var(--muted); margin-top: 8px; } +.stat b { color: var(--text); } + +/* ---- per-model paper-vs-reproduced table ----------------------------------*/ +table.repro { width: 100%; border-collapse: collapse; margin-top: 10px; font-size: 12px; } +table.repro td { padding: 6px 8px; border-bottom: 1px solid var(--line); vertical-align: top; } +table.repro td:first-child { color: var(--muted); width: 92px; white-space: nowrap; } +table.repro td.match-match { color: var(--real); } +table.repro td.match-gap-closed { color: var(--real); font-weight: 600; } +table.repro td.match-below, table.repro td.match-partial, table.repro td.match-ranking-only { color: #ffcf6e; } +table.repro td.match-pending { color: var(--muted); font-style: italic; } +table.repro td.rnote { color: var(--muted); font-size: 11px; font-style: italic; } + +/* ---- full per-benchmark results table -------------------------------------*/ +.repro-full { width: 100%; margin-top: 8px; border-collapse: collapse; font-size: 12px; } +.repro-full th { text-align: left; border-bottom: 1px solid var(--line-strong); padding: 4px 7px; color: var(--muted); } +.repro-full td { padding: 4px 7px; border-bottom: 1px solid var(--line); vertical-align: top; } +.status-reproduced { color: var(--real); font-weight: 600; } +.status-official { color: var(--accent); font-weight: 600; } +.status-below { color: #ffb74d; } +.status-missing { color: var(--muted2); } +.status-reference { color: var(--muted2); font-style: italic; } +.repro-wrap { margin: 10px 0 4px; } +.repro-details { border: 1px solid var(--line); border-radius: var(--radius-sm); background: var(--panel2); overflow: hidden; } +.repro-details > summary { cursor: pointer; list-style: none; padding: 10px 13px; font-size: 12px; color: var(--accent); font-weight: 600; user-select: none; display: flex; align-items: center; gap: 7px; } +.repro-details > summary::-webkit-details-marker { display: none; } +.repro-details > summary::before { content: "▸"; font-size: 10px; transition: transform .15s; } +.repro-details[open] > summary::before { transform: rotate(90deg); } +.repro-scroll { max-height: 360px; overflow: auto; border-top: 1px solid var(--line); } +.repro-full thead th { position: sticky; top: 0; z-index: 1; background: var(--panel-solid); white-space: nowrap; } +.repro-full td:first-child { min-width: 170px; } +.repro-full tbody tr:nth-child(even) { background: rgba(255,255,255,.02); } +.repro-full tbody tr:hover { background: rgba(124, 140, 255, .07); } +.repro-legend { color: var(--muted); font-size: 11px; margin: 8px 12px 10px; line-height: 1.55; } +.lead-head { display: flex; flex-wrap: wrap; gap: 14px; align-items: flex-end; justify-content: space-between; margin: 6px 0 16px; } +.lead-head .muted { flex: 1; min-width: 280px; color: var(--muted); } +.lead-head label { color: var(--muted); font-size: 13px; display: flex; gap: 6px; align-items: center; } +.lead-head select { width: auto; min-width: 220px; } +.lead-card { margin-bottom: 18px; } +.card.lead-card h2, .topgen .card h2 { display: flex; align-items: center; gap: 8px; text-transform: none; letter-spacing: 0; font-size: 17px; color: var(--text); } +.lead-bn { color: var(--muted); font-size: 10px; margin-top: 2px; } +.lb-best { color: var(--accent2); font-weight: 700; } +table.repro-full.lb-gen td { white-space: nowrap; } +table.repro-full.lb-gen td:nth-child(2) { white-space: normal; } +.lb-top td { background: rgba(124, 140, 255, .08); } +.topgen .card { margin-bottom: 18px; } + +/* ---- SOTA generator vs detector comparison --------------------------------*/ +.sota .sota-controls { display: flex; gap: 14px; flex-wrap: wrap; align-items: center; margin: 10px 0; } +.sota .sota-controls label { display: flex; gap: 6px; align-items: center; } +.sota .promptline input { width: 320px; } +.sota .upbtn { cursor: pointer; text-decoration: underline; color: var(--accent); } +.sota .detpick { display: flex; gap: 14px; flex-wrap: wrap; margin: 8px 0; } +.sota .detpick label { display: flex; gap: 4px; align-items: center; font-size: 13px; } +.sota .sota-images { display: flex; gap: 16px; margin: 14px 0; flex-wrap: wrap; } +.sota .sota-images figure { margin: 0; } +.sota .sota-images img { max-width: 320px; max-height: 320px; border: 1px solid var(--line-strong); border-radius: var(--radius-sm); } + +/* ---- weight-source badge --------------------------------------------------*/ +.wtag { font-size: 10px; font-weight: 700; padding: 1px 7px; border-radius: 8px; border: 1px solid var(--line-strong); color: var(--muted); white-space: nowrap; vertical-align: middle; } +.wtag-official { background: rgba(46, 125, 82, .22); border-color: #2e7d52; color: #8ee0b0; } +.wtag-synthscope_trained { background: rgba(46, 93, 125, .22); border-color: #2e5d7d; color: #8ec6e0; } +.wtag-training_free { background: rgba(125, 102, 46, .22); border-color: #7d662e; color: #e0cf8e; } + +/* ---- What's Trending ------------------------------------------------------*/ +.trending h3 { margin: 22px 0 10px; font-size: 15px; color: var(--muted); text-transform: uppercase; letter-spacing: .08em; } +.trend-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(330px, 1fr)); gap: 14px; } +.trend-card { + border: 1px solid var(--line); border-radius: var(--radius-sm); padding: 14px 16px; + background: var(--panel); box-shadow: var(--shadow); transition: transform .15s, border-color .15s; +} +.trend-card:hover { transform: translateY(-3px); border-color: var(--line-strong); } +.trend-head { display: flex; gap: 8px; align-items: baseline; justify-content: space-between; } +.trend-head a { font-weight: 700; color: var(--text); text-decoration: none; } +.trend-head a:hover { color: var(--accent2); } +.trend-links { margin-top: 10px; display: flex; gap: 14px; font-size: 12px; } +.trend-links a { color: var(--accent); text-decoration: none; } +.trend-links a:hover { text-decoration: underline; } +.trend-landscape li { margin: 5px 0; } + +/* links + scrollbars --------------------------------------------------------*/ +a { color: var(--accent); } +::-webkit-scrollbar { width: 10px; height: 10px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: var(--line-strong); border-radius: 8px; border: 2px solid transparent; background-clip: padding-box; } +::-webkit-scrollbar-thumb:hover { background: var(--accent); background-clip: padding-box; } + +/* ---- interpretability map -------------------------------------------------*/ +.interp { margin-top: 16px; } +.interp-row { display: flex; gap: 12px; margin-top: 8px; flex-wrap: wrap; } +.interp-cell { margin: 0; } +.interp-cell figcaption { font-size: 11px; color: #8b93a7; margin-top: 4px; text-align: center; } +.interp-map { display: block; width: 100%; max-width: 240px; border-radius: var(--radius-sm); + border: 1px solid var(--line-strong); box-shadow: var(--shadow); } diff --git a/interface/frontend/vite.config.js b/interface/frontend/vite.config.js new file mode 100644 index 0000000000000000000000000000000000000000..2114ca11fd039ac27a349266b5ffeae47b8fcc46 --- /dev/null +++ b/interface/frontend/vite.config.js @@ -0,0 +1,21 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +// In dev, proxy /api to the FastAPI backend (default :8000). The production build emits to +// ../frontend/dist, which the backend serves directly. +export default defineConfig({ + plugins: [react()], + server: { + port: 5173, + proxy: { + "/api": { + target: process.env.SYNTHSCOPE_BACKEND || "http://127.0.0.1:8000", + changeOrigin: true, + }, + }, + }, + build: { + outDir: "dist", + emptyOutDir: true, + }, +}); diff --git a/interface/requirements.txt b/interface/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b786c6e21b91e366506691320fc35b1b88a1879 --- /dev/null +++ b/interface/requirements.txt @@ -0,0 +1,5 @@ +# Web-interface backend deps (installed on top of the existing `SynthScope` conda env, which +# already provides torch, torchvision, decord, clip, open_clip, diffusers, hydra, PIL, ...). +fastapi>=0.110 +uvicorn[standard]>=0.29 +python-multipart>=0.0.9 diff --git a/interface/run_backend.sh b/interface/run_backend.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c0f3497d1d5a2f9cf274b188d9fda08800c3a8f --- /dev/null +++ b/interface/run_backend.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Launch the SynthScope interface backend (FastAPI + uvicorn) from the SynthScope conda env. +# +# bash interface/run_backend.sh # serve on 0.0.0.0:8000 +# SYNTHSCOPE_PORT=9000 bash interface/run_backend.sh # custom port +# +# The backend serves the built Vite frontend (interface/frontend/dist) if present, else the +# no-build standalone page (interface/static/index.html). Models load lazily on first request. +set -euo pipefail + +# Old AGID_* variables still work: map each to its SYNTHSCOPE_* name unless that is already set. +for _v in $(compgen -e | grep '^AGID_' || true); do _n="SYNTHSCOPE_${_v#AGID_}"; [ -n "${!_n:-}" ] || export "$_n=${!_v}"; done + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +PORT="${SYNTHSCOPE_PORT:-8000}" +HOST="${SYNTHSCOPE_HOST:-0.0.0.0}" +CONDA="${CONDA_EXE:-/home/ssalehi/miniconda3/bin/conda}" + +# Caches default to a node-local location (see backend/config.py). Override as needed: +# export SYNTHSCOPE_CKPT_DIR=/path/to/checkpoints + +echo "Serving SynthScope interface on http://${HOST}:${PORT} (Ctrl-C to stop)" +exec "$CONDA" run --no-capture-output -n SynthScope \ + uvicorn interface.backend.app:app --host "$HOST" --port "$PORT" "$@" diff --git a/interface/scripts/_finalize_restrav.py b/interface/scripts/_finalize_restrav.py new file mode 100644 index 0000000000000000000000000000000000000000..216f5b7c57195bd47e1589a599992981dca8cae1 --- /dev/null +++ b/interface/scripts/_finalize_restrav.py @@ -0,0 +1,19 @@ +"""Read the ReStraV eval log, write the deployed cross-generator result, and refresh the README number.""" +import re, sys +from pathlib import Path +log = Path(sys.argv[1]).read_text(errors="ignore") +lines = [l for l in log.splitlines() if l.startswith("restrav: acc@0.5=")] +if not lines: + print("[finalize] no eval line found"); sys.exit(0) +last = lines[-1] +ap = re.search(r"AP=([\d.]+)", last).group(1) +auc = re.search(r"AUC=([\d.]+)", last).group(1) +res = Path("/hddstore/ssalehi/SynthScope/checkpoints/_train/restrav_exact_RESULT.txt") +res.write_text(f"deployed ReStraV (exact official recipe) cross-generator on full GenVideo-Val:\n AP {ap} / AUROC {auc}\n ({last})\n") +readme = Path("README.md") +t = readme.read_text() +t2 = re.sub(r"\*\*AP [\d.]+ / AUROC [\d.]+\*\* \(last run\)", f"**AP {ap} / AUROC {auc}** (last run)", t, count=1) +if t2 != t: + readme.write_text(t2); print(f"[finalize] README updated -> AP {ap} / AUROC {auc}") +else: + print("[finalize] README token not found (left unchanged)") diff --git a/interface/scripts/build_restrav_data.py b/interface/scripts/build_restrav_data.py new file mode 100644 index 0000000000000000000000000000000000000000..1c80f5f76deef553b47a95e46c91ff3d4ca46819 --- /dev/null +++ b/interface/scripts/build_restrav_data.py @@ -0,0 +1,111 @@ +"""Stage the local d3_repro GenVideo data into the VideoFolder layout ReStraV training needs: + + //genvideo/{0_real,1_fake}/*.mp4 (symlinks) + +Sources (already on cn023): + * train / val : d3_repro/extracted/*.mp4 (flat; filename `__.mp4`, + gen == 'real' => real/label 0, else fake/label 1) + * test : d3_repro/genvideo_val/GenVideo-Val/{Real, Fake/} (GenVideo-Val) + +The in-loop ``test`` split is a balanced SUBSAMPLE (videos are re-decoded every epoch, so the +full 18k-clip benchmark would dominate runtime). The full GenVideo-Val also gets staged under +``_fulltest/test`` for a faithful final evaluation (see eval_restrav_full.py). + +Single 'genvideo' folder per split: with one folder_id the per-folder AP/AUROC AIGIMetrics +reports equal the global real-vs-fake metric (per-generator folders would be single-class). + + conda run -n SynthScope python interface/scripts/build_restrav_data.py +Override OUT with SYNTHSCOPE_RESTRAV_DIR. +""" +import os +import random +import re +import shutil +from collections import defaultdict +from pathlib import Path + +EXTRACTED = Path("/hddstore/ssalehi/SynthScope/d3_repro/extracted") +GENVIDEO_VAL = Path("/hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val") +OUT = Path(os.environ.get("SYNTHSCOPE_RESTRAV_DIR", "/hddstore/ssalehi/SynthScope/restrav_data")) +FULLTEST = Path(str(OUT) + "_fulltest") + +N_TRAIN_PER_CLASS = int(os.environ.get("RESTRAV_N_TRAIN", "8000")) +N_VAL_PER_CLASS = int(os.environ.get("RESTRAV_N_VAL", "500")) +N_TEST_PER_CLASS = int(os.environ.get("RESTRAV_N_TEST", "400")) # balanced in-loop test subsample +SEED = 42 +NAME_RE = re.compile(r"^\d+_(.+?)_[^_]+\.mp4$") + + +def gen_of(name: str) -> str: + m = NAME_RE.match(name) + if m: + return m.group(1) + stem = name[:-4] + parts = stem.split("_", 1) + return parts[1].split("-")[0] if len(parts) > 1 else "unknown" + + +def link(src: Path, dst: Path): + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.exists() or dst.is_symlink(): + dst.unlink() + os.symlink(src.resolve(), dst) + + +def stage_train_val(): + reals, fakes = [], [] + for p in sorted(EXTRACTED.glob("*.mp4")): + (reals if gen_of(p.name) == "real" else fakes).append(p) + random.Random(SEED).shuffle(reals) + random.Random(SEED + 1).shuffle(fakes) + splits = { + "train": (reals[:N_TRAIN_PER_CLASS], fakes[:N_TRAIN_PER_CLASS]), + "val": (reals[N_TRAIN_PER_CLASS:N_TRAIN_PER_CLASS + N_VAL_PER_CLASS], + fakes[N_TRAIN_PER_CLASS:N_TRAIN_PER_CLASS + N_VAL_PER_CLASS]), + } + for split, (rl, fk) in splits.items(): + for p in rl: + link(p, OUT / split / "genvideo" / "0_real" / p.name) + for p in fk: + link(p, OUT / split / "genvideo" / "1_fake" / p.name) + print(f"{split}: real={len(rl)} fake={len(fk)}") + + +def _genvideo_real_fake(): + reals = sorted(GENVIDEO_VAL.glob("Real/**/*.mp4")) + fakes = [] + for gen_dir in sorted((GENVIDEO_VAL / "Fake").iterdir()): + if gen_dir.is_dir(): + fakes += sorted(gen_dir.glob("**/*.mp4")) + return reals, fakes + + +def stage_test(): + reals, fakes = _genvideo_real_fake() + # full test (faithful eval) + for p in reals: + link(p, FULLTEST / "test" / "genvideo" / "0_real" / p.name) + for p in fakes: + link(p, FULLTEST / "test" / "genvideo" / "1_fake" / p.name) + print(f"fulltest: real={len(reals)} fake={len(fakes)}") + # balanced subsample for the in-loop test + random.Random(SEED + 2).shuffle(reals) + random.Random(SEED + 3).shuffle(fakes) + for p in reals[:N_TEST_PER_CLASS]: + link(p, OUT / "test" / "genvideo" / "0_real" / p.name) + for p in fakes[:N_TEST_PER_CLASS]: + link(p, OUT / "test" / "genvideo" / "1_fake" / p.name) + print(f"test(subsample): real={min(len(reals),N_TEST_PER_CLASS)} fake={min(len(fakes),N_TEST_PER_CLASS)}") + + +def main(): + for d in (OUT, FULLTEST): + if d.exists(): + shutil.rmtree(d) + stage_train_val() + stage_test() + print("staged into", OUT, "and", FULLTEST) + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/calibrate_d3.py b/interface/scripts/calibrate_d3.py new file mode 100644 index 0000000000000000000000000000000000000000..9f81daf4ea852e38f9c5e5867917c94b3367d3bd --- /dev/null +++ b/interface/scripts/calibrate_d3.py @@ -0,0 +1,89 @@ +"""Calibrate the training-free D3 video detector so its output is a usable probability. + +D3's raw score is the per-video second-order feature volatility (std). The model's argmax +threshold defaults to 0.0, which is not a real decision boundary, so every video reads as +"real". This script estimates, on a balanced real/fake sample of GenVideo-Val: + * threshold -- midpoint between the class-mean scores, + * scale -- pooled std (logistic temperature), and + * higher_is_fake -- which class has the larger score, +and writes them to ``/d3_calibration.json``. The engine then reports +``sigmoid(sign * (score - threshold) / scale)`` as P(fake). + + conda run -n SynthScope python interface/scripts/calibrate_d3.py [--n 200] +""" +import argparse +import json +import math +import random +import sys +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend import config # noqa: E402 +from backend.engine import get_engine # noqa: E402 +from backend.registry import get_spec, CLIP_MEAN, CLIP_STD # noqa: E402 +from backend import media # noqa: E402 + +GENVIDEO = Path("/hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val") + + +def collect(n): + reals = sorted((GENVIDEO / "Real").rglob("*.mp4")) + fakes = sorted((GENVIDEO / "Fake").rglob("*.mp4")) + random.Random(0).shuffle(reals) + random.Random(1).shuffle(fakes) + return reals[:n], fakes[:n] + + +@torch.no_grad() +def score_paths(engine, spec, model, paths): + mean = torch.tensor(CLIP_MEAN).view(1, 3, 1, 1) + std = torch.tensor(CLIP_STD).view(1, 3, 1, 1) + out = [] + for p in paths: + try: + clip = media.video_clip_tensor(p, spec.num_frames, spec.fps, spec.frame_size) + clip = ((clip - mean) / std).unsqueeze(0).to(engine.device) + out.append(float(model.score(clip).item())) + except Exception as e: + print(f" skip {p.name}: {e}") + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--n", type=int, default=200) + args = ap.parse_args() + + engine = get_engine() + spec = get_spec("d3") + model = engine._get_model(spec) + reals, fakes = collect(args.n) + print(f"scoring {len(reals)} real + {len(fakes)} fake clips on {engine.device}...") + rs = score_paths(engine, spec, model, reals) + fs = score_paths(engine, spec, model, fakes) + + mr, mf = sum(rs) / len(rs), sum(fs) / len(fs) + var = (sum((x - mr) ** 2 for x in rs) + sum((x - mf) ** 2 for x in fs)) / (len(rs) + len(fs)) + scale = max(math.sqrt(var), 1e-6) + threshold = (mr + mf) / 2.0 + higher_is_fake = mf > mr + + # quick sanity accuracy at this threshold + def pred_fake(s): + return (s > threshold) if higher_is_fake else (s < threshold) + acc = (sum(not pred_fake(s) for s in rs) + sum(pred_fake(s) for s in fs)) / (len(rs) + len(fs)) + + cal = {"threshold": threshold, "scale": scale, "higher_is_fake": higher_is_fake, + "mean_real": mr, "mean_fake": mf, "n": args.n, "balanced_acc": acc} + out = config.CHECKPOINTS_DIR / "d3_calibration.json" + out.write_text(json.dumps(cal, indent=2)) + print(f"mean_real={mr:.5f} mean_fake={mf:.5f} threshold={threshold:.5f} scale={scale:.5f} " + f"higher_is_fake={higher_is_fake} acc={acc:.3f}") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/collect_checkpoints.py b/interface/scripts/collect_checkpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..8a5b97fe32941532a45c6a234201492ccc07fe1b --- /dev/null +++ b/interface/scripts/collect_checkpoints.py @@ -0,0 +1,79 @@ +"""Assemble the central checkpoint directory used by the web interface. + +- Symlinks the already-available OFFICIAL checkpoints (FatFormer, AIDE, NPR) to the canonical + filenames the registry expects. +- Writes a MANIFEST.{json,md} summarizing, per model, whether a checkpoint is present and how + it was obtained (official weights / trained-from-scratch / training-free). + +Run: conda run -n SynthScope python interface/scripts/collect_checkpoints.py +Override the destination with SYNTHSCOPE_CKPT_DIR. +""" +import json +import os +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend import config # noqa: E402 (sets paths) +from backend.registry import list_specs # noqa: E402 + +# Known source locations of the official weights on this node. +REPRO_CKPTS = Path("/hddstore/ssalehi/SynthScope/repro/ckpts") +OFFICIAL_SOURCES = { + "fat_former.pth": REPRO_CKPTS / "fatformer_official.pth", + "aide.pth": REPRO_CKPTS / "aide_progan_train.pth", + "npr.pth": REPRO_CKPTS / "NPR_model_epoch_last_3090.pth", +} + + +def link(dest: Path, src: Path) -> str: + if not src.exists(): + return f"source missing: {src}" + if dest.exists() or dest.is_symlink(): + dest.unlink() + dest.symlink_to(src) + return f"linked -> {src}" + + +def main(): + dst = config.CHECKPOINTS_DIR + dst.mkdir(parents=True, exist_ok=True) + print(f"checkpoints dir: {dst}") + + for name, src in OFFICIAL_SOURCES.items(): + print(f" {name:18s} {link(dst / name, src)}") + + manifest = {"checkpoints_dir": str(dst), "generated": time.strftime("%Y-%m-%d %H:%M:%S"), + "models": []} + for spec in list_specs(): + if spec.ckpt_kind == "none": + status, present = "training-free", True + else: + p = dst / spec.checkpoint + present = p.exists() + if spec.ckpt_kind.startswith("official"): + status = "official weights" if present else "MISSING (official)" + else: + status = "trained from scratch" if present else "MISSING (needs training)" + manifest["models"].append({ + "key": spec.key, "name": spec.name, "modality": spec.modality, + "checkpoint": spec.checkpoint, "present": present, "status": status, + }) + + (dst / "MANIFEST.json").write_text(json.dumps(manifest, indent=2)) + lines = ["# SynthScope checkpoints manifest", "", + f"Directory: `{dst}` ", f"Generated: {manifest['generated']}", "", + "| Model | Modality | Checkpoint | Present | Status |", + "| --- | --- | --- | --- | --- |"] + for m in manifest["models"]: + lines.append(f"| {m['key']} | {m['modality']} | {m['checkpoint'] or '—'} | " + f"{'✅' if m['present'] else '❌'} | {m['status']} |") + (dst / "MANIFEST.md").write_text("\n".join(lines) + "\n") + print(f"\nwrote {dst/'MANIFEST.json'} and {dst/'MANIFEST.md'}") + present = sum(m["present"] for m in manifest["models"]) + print(f"present: {present}/{len(manifest['models'])} models") + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/d3_diag.py b/interface/scripts/d3_diag.py new file mode 100644 index 0000000000000000000000000000000000000000..83e8a9155e9c53df07733ecdd97ff2c88b025813 --- /dev/null +++ b/interface/scripts/d3_diag.py @@ -0,0 +1,58 @@ +"""Diagnose why D3 calls everything fake on GenVidBench but works on EvalCrafter. + +D3 score = std of 2nd-order frozen-CLIP trajectory features; calibration says real>thr(5.68)>fake. +We print the raw score, calibrated P(fake), and the clip duration/native-fps for a few videos in +each group, to see whether GenVidBench reals score low (flat) and whether it's a short-clip artifact. +""" +import glob, random, sys +from pathlib import Path +import numpy as np, torch, decord + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend.engine import get_engine # noqa: E402 +from backend.registry import get_spec # noqa: E402 +from backend import media # noqa: E402 + +GVB = "/hddstore/ssalehi/SynthScope/GenVidBench/test" +EVC = "/hddstore/ssalehi/SynthScope/EvalCrafter/test" +GROUPS = { + "GVB real (vript+hd_vg)": sorted(glob.glob(f"{GVB}/0_real/*/*.mp4")), + "GVB fake (8 gens)": sorted(glob.glob(f"{GVB}/1_fake/*/*.mp4")), + "EvalCrafter fake": sorted(glob.glob(f"{EVC}/1_fake/*/*.mp4")), +} + + +def native_dur(path): + vr = decord.VideoReader(path, ctx=decord.cpu(0)) + n = len(vr); fps = float(vr.get_avg_fps()) or 30.0 + return n, fps, n / fps + + +@torch.no_grad() +def main(): + eng = get_engine(); spec = get_spec("d3"); model = eng._get_model(spec) + mean = torch.tensor(spec.video_norm[0]).view(1, 3, 1, 1) + std = torch.tensor(spec.video_norm[1]).view(1, 3, 1, 1) + for gname, paths in GROUPS.items(): + random.Random(0).shuffle(paths) + scores, probs, durs, nfr = [], [], [], [] + for p in paths[:20]: + try: + n, fps, dur = native_dur(p) + clip = media.video_clip_tensor(p, spec.num_frames, spec.fps, spec.frame_size, + window_center=spec.window_center, interp=spec.resize_interp) + clip = (clip - mean) / std + s = float(model.score(clip.unsqueeze(0).to(eng.device)).item()) + pf = float(eng._video_fake_prob(model, spec, clip).item()) + scores.append(s); probs.append(pf); durs.append(dur); nfr.append(n) + except Exception as e: + print(f" skip {Path(p).name}: {e}") + scores, probs, durs = map(np.array, (scores, probs, durs)) + print(f"\n{gname}: n={len(scores)}") + print(f" D3 raw score mean={scores.mean():.2f} (real>5.68>fake; min={scores.min():.2f} max={scores.max():.2f})") + print(f" P(fake) mean={probs.mean():.3f} -> called fake: {(probs>0.5).mean()*100:.0f}%") + print(f" native dur(s) mean={durs.mean():.2f} min={durs.min():.2f} max={durs.max():.2f} nframes mean={np.mean(nfr):.0f}") + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/eval_restrav_full.py b/interface/scripts/eval_restrav_full.py new file mode 100644 index 0000000000000000000000000000000000000000..b8eba71474e45d0a908f76c0bef8e7504d12d086 --- /dev/null +++ b/interface/scripts/eval_restrav_full.py @@ -0,0 +1,79 @@ +"""Evaluate the ReStraV video detector on the full (staged) GenVideo-Val test set, through the +same engine the web app uses. Reports acc@0.5, real/fake accuracy, AP and AUROC. + + conda run -n SynthScope python interface/scripts/eval_restrav_full.py [--n 800] +--n caps videos PER CLASS (videos are decode-heavy); 0 = all. +""" +import argparse +import random +import sys +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend.engine import get_engine # noqa: E402 +from backend.registry import get_spec # noqa: E402 +from backend import media # noqa: E402 + +ROOT = Path("/hddstore/ssalehi/SynthScope/restrav_data_fulltest/test/genvideo") + + +def ap_auc(y, p): + y = np.asarray(y); p = np.asarray(p) + order = np.argsort(-p, kind="mergesort"); y = y[order] + tp = np.cumsum(y == 1); fp = np.cumsum(y == 0) + npos, nneg = (y == 1).sum(), (y == 0).sum() + if npos == 0 or nneg == 0: + return float("nan"), float("nan") + prec = tp / np.maximum(tp + fp, 1); rec = tp / npos + rec = np.concatenate([[0.0], rec]); prec = np.concatenate([[1.0], prec]) + ap = float(np.sum((rec[1:] - rec[:-1]) * prec[1:])) + tpr = np.concatenate([[0.0], tp / npos]); fpr = np.concatenate([[0.0], fp / nneg]) + auc = float(np.sum((fpr[1:] - fpr[:-1]) * (tpr[1:] + tpr[:-1]) / 2.0)) + return ap, auc + + +@torch.no_grad() +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--n", type=int, default=800) + args = ap.parse_args() + engine = get_engine() + spec = get_spec("restrav") + model = engine._get_model(spec) + mean = torch.tensor(spec.video_norm[0]).view(1, 3, 1, 1) + std = torch.tensor(spec.video_norm[1]).view(1, 3, 1, 1) + + reals = sorted((ROOT / "0_real").glob("*.mp4")) + fakes = sorted((ROOT / "1_fake").glob("*.mp4")) + if args.n > 0: + random.Random(0).shuffle(reals); random.Random(1).shuffle(fakes) + reals, fakes = reals[:args.n], fakes[:args.n] + items = [(p, 0) for p in reals] + [(p, 1) for p in fakes] + print(f"restrav full eval: {len(reals)} real + {len(fakes)} fake on {engine.device}") + + probs, labels = [], [] + for i, (p, y) in enumerate(items): + try: + clip = media.video_clip_tensor(p, spec.num_frames, spec.fps, spec.frame_size, + window_center=spec.window_center, interp=spec.resize_interp) + clip = ((clip - mean) / std) + probs.append(float(engine._video_fake_prob(model, spec, clip).item())) + labels.append(y) + except Exception as e: + print(f" skip {p.name}: {e}") + if (i + 1) % 200 == 0: + print(f" {i+1}/{len(items)}") + probs = np.array(probs); labels = np.array(labels) + pred = (probs > 0.5).astype(int) + acc = float((pred == labels).mean()) + racc = float((pred[labels == 0] == 0).mean()); facc = float((pred[labels == 1] == 1).mean()) + apv, aucv = ap_auc(labels, probs) + print(f"restrav: acc@0.5={acc*100:.2f} real_acc={racc*100:.2f} fake_acc={facc*100:.2f} " + f"AP={apv*100:.2f} AUC={aucv*100:.2f}") + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/eval_restrav_splits.py b/interface/scripts/eval_restrav_splits.py new file mode 100644 index 0000000000000000000000000000000000000000..b4b4b5751298eaff43effd4910430849a64e39d6 --- /dev/null +++ b/interface/scripts/eval_restrav_splits.py @@ -0,0 +1,70 @@ +"""Evaluate the DEPLOYED ReStraV (restrav.pth, the serving path) on restrav_data val vs test, +to check whether the 'awful test' performance matches the AUROC numbers we've quoted. + +val = in-distribution (VidProM: VSC reals + VidProM generators) -> expect high. +test = cross-dataset (GenVideo: msrvtt reals + GenVideo generators) -> expect ~chance. +""" +import glob, random, sys +from pathlib import Path +import numpy as np, torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend.engine import get_engine # noqa: E402 +from backend.registry import get_spec # noqa: E402 +from backend import media # noqa: E402 + +ROOT = "/hddstore/ssalehi/SynthScope/restrav_data" +N = 200 # per class per split + + +def auroc(y, p): + y = np.asarray(y); p = np.asarray(p) + order = np.argsort(-p, kind="mergesort"); y = y[order] + tp = np.cumsum(y == 1); fp = np.cumsum(y == 0) + npos, nneg = (y == 1).sum(), (y == 0).sum() + if npos == 0 or nneg == 0: + return float("nan") + tpr = np.concatenate([[0.0], tp / npos]); fpr = np.concatenate([[0.0], fp / nneg]) + return float(np.sum((fpr[1:] - fpr[:-1]) * (tpr[1:] + tpr[:-1]) / 2.0)) + + +@torch.no_grad() +def main(): + eng = get_engine(); spec = get_spec("restrav"); model = eng._get_model(spec) + mean = torch.tensor(spec.video_norm[0]).view(1, 3, 1, 1) + std = torch.tensor(spec.video_norm[1]).view(1, 3, 1, 1) + for split in ["val", "test"]: + reals = sorted(glob.glob(f"{ROOT}/{split}/genvideo/0_real/*.mp4")) + fakes = sorted(glob.glob(f"{ROOT}/{split}/genvideo/1_fake/*.mp4")) + random.Random(0).shuffle(reals); random.Random(1).shuffle(fakes) + items = [(p, 0) for p in reals[:N]] + [(p, 1) for p in fakes[:N]] # SynthScope: real=0, fake=1 + probs, labels = [], [] + for p, y in items: + try: + clip = media.video_clip_tensor(p, spec.num_frames, spec.fps, spec.frame_size, + window_center=spec.window_center, interp=spec.resize_interp) + clip = (clip - mean) / std + probs.append(float(eng._video_fake_prob(model, spec, clip).item())); labels.append(y) + except Exception: + pass + probs = np.array(probs); labels = np.array(labels) + pred = (probs > 0.5).astype(int) + racc = (pred[labels == 0] == 0).mean() if (labels == 0).any() else float("nan") + facc = (pred[labels == 1] == 1).mean() if (labels == 1).any() else float("nan") + # AUROC with fake as positive (label 1 = fake, score = P(fake)) + a = auroc((labels == 1).astype(int), probs) + acc = (pred == labels).mean() * 100 + # best-threshold balanced accuracy (what "Auto-calibrate" finds, fit IN-SAMPLE) + best_bacc, best_t = 0.0, 0.5 + for t in np.linspace(0.01, 0.99, 99): + p2 = (probs > t).astype(int) + r = (p2[labels == 0] == 0).mean() if (labels == 0).any() else 0 + f = (p2[labels == 1] == 1).mean() if (labels == 1).any() else 0 + if (r + f) / 2 > best_bacc: + best_bacc, best_t = (r + f) / 2, t + print(f"{split:5s} (n={len(labels)}): acc@0.5={acc:.1f} real_acc={racc*100:.1f} fake_acc={facc*100:.1f} " + f"AUROC={a*100:.1f} | best-threshold(t={best_t:.2f}) balanced_acc={best_bacc*100:.1f}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/promote_checkpoint.py b/interface/scripts/promote_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..b68b2a73dc44f855c911bc30db7fc5b1fc4a5f00 --- /dev/null +++ b/interface/scripts/promote_checkpoint.py @@ -0,0 +1,60 @@ +"""Publish a trained checkpoint into the central directory under its canonical name. + +The trainer writes ``model_.pth`` to the staging dir on each val-loss improvement +(with save_policy=min the last/highest-epoch file is the best). This copies that file to +``.pth`` so the interface registry picks it up, then refreshes the manifest. + + conda run -n SynthScope python interface/scripts/promote_checkpoint.py [--epoch N] [--copy] +""" +import argparse +import re +import shutil +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend import config # noqa: E402 +from backend.registry import get_spec # noqa: E402 + + +def best_epoch_file(stage_dir: Path, epoch: int | None) -> Path: + files = list(stage_dir.glob("model_*.pth")) + if not files: + raise FileNotFoundError(f"No model_*.pth in {stage_dir}") + if epoch is not None: + f = stage_dir / f"model_{epoch}.pth" + if not f.exists(): + raise FileNotFoundError(f"{f} not found") + return f + # Most recently written = the current run's best (save_policy=min writes the best last). + # Use mtime, NOT the epoch number, so stale higher-epoch checkpoints left in the staging dir + # by a *previous* run (possibly a different architecture) are never picked. + return max(files, key=lambda p: p.stat().st_mtime) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("key") + ap.add_argument("--epoch", type=int, default=None) + ap.add_argument("--copy", action="store_true", help="copy instead of symlink") + args = ap.parse_args() + + spec = get_spec(args.key) + if spec.checkpoint is None: + print(f"{args.key} is training-free; nothing to promote.") + return + stage = config.CHECKPOINTS_DIR / "_train" / args.key + src = best_epoch_file(stage, args.epoch) + dst = config.CHECKPOINTS_DIR / spec.checkpoint + if dst.exists() or dst.is_symlink(): + dst.unlink() + if args.copy: + shutil.copy2(src, dst) + print(f"copied {src} -> {dst}") + else: + dst.symlink_to(src) + print(f"linked {dst} -> {src}") + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/restrav_repro.py b/interface/scripts/restrav_repro.py new file mode 100644 index 0000000000000000000000000000000000000000..89b068890f92d439976ba57ea7944fb77d98c6c5 --- /dev/null +++ b/interface/scripts/restrav_repro.py @@ -0,0 +1,122 @@ +"""Faithful ReStraV reproduction (mirrors github.com/ChristianInterno/ReStraV). + +Matches the official pipeline exactly, except torchcodec (unavailable here) is replaced by +decord with identical frame selection: + * decode: 24 frames over a CENTER 2s window, repeat_last clamp (dinov2_features.decode_clip) + * preprocess: float/255 + BICUBIC resize to 224, NO ImageNet normalization (dinov2_features.preprocess) + * DINOv2 ViT-S/14 forward_features -> [CLS ; patches] flattened -> Z[T,D] (extract_dinov2_embeddings) + * 21-d features: d[:7], theta[:6], [mu_d,min_d,max_d,var_d, mu_t,min_t,max_t,var_t] (features_from_Z) + * label REAL=1, FAKE=0 (extract_training_features) + * standardize with train mean/std; MLP 21->64->32->1, BCEWithLogits, Adam 1e-3, 10 epochs (train.py) + * report ROC-AUC (and AP for fake=positive) + +Runs TWO protocols: + IN-DISTRIBUTION : GenVideo-Val pool, balanced, random 50/50 stratified split (shared generators) + CROSS-GENERATOR : train on d3_repro/extracted generators, test on disjoint GenVideo-Val generators +""" +import os, sys, glob, re, random +os.environ.setdefault("HF_HOME", "/hddstore/ssalehi/SynthScope/.cache/huggingface") +os.environ.setdefault("TORCH_HOME", "/hddstore/ssalehi/SynthScope/.cache/torch") +import numpy as np, torch, torch.nn as nn, torch.nn.functional as F +import torchvision.transforms.functional as TF +from torchvision.transforms import InterpolationMode +import decord +from sklearn.model_selection import train_test_split +from sklearn.metrics import roc_auc_score, average_precision_score + +EXTRACTED = "/hddstore/ssalehi/SynthScope/d3_repro/extracted" +GV = "/hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val" +DEV = "cuda" +N = int(os.environ.get("RESTRAV_REPRO_N", "500")) # per class, per pool +T, WIN = 24, 2.0 + +dinov2 = torch.hub.load("facebookresearch/dinov2", "dinov2_vits14").eval().to(DEV) + + +def decode_clip(path): + vr = decord.VideoReader(path, ctx=decord.cpu(0)) + n = len(vr); fps = float(vr.get_avg_fps()) or 30.0; dur = n / fps + center = dur / 2.0; start = max(0.0, center - WIN / 2) + delta = WIN / max(T - 1, 1) + idxs = [min(int(round((start + i * delta) * fps)), n - 1) for i in range(T)] # repeat_last clamp + frames = vr.get_batch(idxs).asnumpy() # [T,H,W,3] uint8 + return torch.from_numpy(frames).permute(0, 3, 1, 2) # [T,3,H,W] + + +def preprocess(frames): + frames = frames.float() / 255.0 + return torch.stack([TF.resize(x, [224, 224], InterpolationMode.BICUBIC, antialias=True) for x in frames]) + + +@torch.no_grad() +def feats_of(path): + clip = preprocess(decode_clip(path)).to(DEV) + f = dinov2.forward_features(clip) + tokens = torch.cat([f["x_norm_clstoken"].unsqueeze(1), f["x_norm_patchtokens"]], dim=1) + Z = tokens.flatten(1).unsqueeze(0) # [1,T,D] + dz = Z[:, 1:] - Z[:, :-1]; d = dz.norm(dim=-1) + cos = F.cosine_similarity(dz[:, :-1], dz[:, 1:], dim=-1) + th = torch.rad2deg(torch.acos(cos.clamp(-1, 1))) + def m4(x): return [x.mean(-1), x.amin(-1), x.amax(-1), x.var(-1, unbiased=False)] + stats = torch.stack(m4(d) + m4(th), dim=1) # [1,8] (mu_d,min,max,var, mu_t,min,max,var) + return torch.cat([d[:, :7], th[:, :6], stats], dim=1)[0].cpu().numpy() # [21] + + +def build(items, tag): + X, y = [], [] + for i, (p, lab) in enumerate(items): + try: + f = feats_of(p) + if np.isfinite(f).all(): X.append(f); y.append(lab) + except Exception: + pass + if (i + 1) % 200 == 0: print(f" [{tag}] {i+1}/{len(items)}", flush=True) + return np.array(X, np.float32), np.array(y, np.int64) + + +def train_eval(Xtr, ytr, Xte, yte, tag): + # standardize with TRAIN stats (official) + mu = Xtr.mean(0, keepdims=True); sd = Xtr.std(0, keepdims=True) + 1e-8 + Xtr = (Xtr - mu) / sd; Xte = (Xte - mu) / sd + net = nn.Sequential(nn.Linear(21, 64), nn.ReLU(), nn.Linear(64, 32), nn.ReLU(), nn.Linear(32, 1)).to(DEV) + opt = torch.optim.Adam(net.parameters(), lr=1e-3); crit = nn.BCEWithLogitsLoss() + Xt = torch.tensor(Xtr, device=DEV); yt = torch.tensor(ytr, dtype=torch.float32, device=DEV).unsqueeze(1) + for ep in range(10): + perm = torch.randperm(len(Xt)) + for i in range(0, len(Xt), 32): + b = perm[i:i+32]; opt.zero_grad() + loss = crit(net(Xt[b]), yt[b]); loss.backward(); opt.step() + net.eval() + with torch.no_grad(): + p_real = torch.sigmoid(net(torch.tensor(Xte, device=DEV))).cpu().numpy().ravel() + # label real=1; AUC on real. For fake-detection AP, positive=fake => score 1-p_real, label (1-yte) + auc = roc_auc_score(yte, p_real) + ap_fake = average_precision_score(1 - yte, 1 - p_real) + print(f" {tag}: ROC-AUC={auc:.3f} AP(fake)={ap_fake:.3f} (n_test={len(yte)})", flush=True) + + +def main(): + random.seed(0) + # GenVideo-Val pool (label real=1, fake=0) + gvr = sorted(glob.glob(GV + "/Real/*.mp4")); gvf = sorted(glob.glob(GV + "/Fake/*/*.mp4")) + random.Random(0).shuffle(gvr); random.Random(1).shuffle(gvf) + print(f"Extracting GenVideo-Val features ({N}/class)...", flush=True) + Xg, yg = build([(p, 1) for p in gvr[:N]] + [(p, 0) for p in gvf[:N]], "GV") + + print("\n=== IN-DISTRIBUTION (GenVideo-Val, random 50/50 split, shared generators) ===") + Xtr, Xte, ytr, yte = train_test_split(Xg, yg, test_size=0.5, stratify=yg, random_state=42) + train_eval(Xtr, ytr, Xte, yte, "in-distribution") + + # extracted pool (label real=1, fake=0) + ex = sorted(glob.glob(EXTRACTED + "/*.mp4")) + exr = [p for p in ex if re.search(r"_real_", os.path.basename(p))] + exf = [p for p in ex if not re.search(r"_real_", os.path.basename(p))] + random.Random(2).shuffle(exr); random.Random(3).shuffle(exf) + print(f"\nExtracting train-pool (extracted) features ({N}/class)...", flush=True) + Xe, ye = build([(p, 1) for p in exr[:N]] + [(p, 0) for p in exf[:N]], "EX") + print("\n=== CROSS-GENERATOR (train=extracted generators, test=disjoint GenVideo-Val) ===") + train_eval(Xe, ye, Xg, yg, "cross-generator") + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/restrav_xgen_repro.py b/interface/scripts/restrav_xgen_repro.py new file mode 100644 index 0000000000000000000000000000000000000000..7f3fe3094f5baf423cb3de4b14fa0f50394b88a2 --- /dev/null +++ b/interface/scripts/restrav_xgen_repro.py @@ -0,0 +1,211 @@ +"""Decisive ReStraV cross-generator reproduction. + +Question: our README reports a weak/unstable ReStraV "cross-generator" number, and the deployed +restrav.pth collapses to AUROC ~0.50. Does the OFFICIAL ReStraV recipe (github.com/ChristianInterno/ +ReStraV) reproduce the paper's cross-generator generalization (Table 3: unseen-generator mAP 97.3, +future/Sora mAP 92.9), or does it also collapse? + +This script extracts the official 21-d trajectory-geometry features (verbatim official recipe: +center 2s / 24 frames / BICUBIC-224 / DINOv2 ViT-S/14 / [CLS;patches]) and runs the official MLP +training recipe (21->64->32->1, BCE, Adam 1e-3, 10 epochs, standardized by train mean/std) under +THREE evaluation protocols, to separate *method/code* faithfulness from *protocol* mismatch: + + (A) IN-DISTRIBUTION official train.py protocol: GenVideo-Val pool, balanced, stratified 50/50. + (B) UNSEEN-GENERATOR paper Table-3 protocol: SAME real source, leave-one-fake-generator-out + (train on the other generators, test on the held-out one). + (C) CROSS-DATASET our README "cross-generator": train on `extracted` (VidProM/DeMamba pool, + VSC reals), test on GenVideo-Val (DIFFERENT real source + different fakes). + +To settle the normalization question, every video's features are computed TWICE in one decode pass: + * raw = official preprocessing (frames/255, NO ImageNet normalization) <- official recipe + * norm = ImageNet-normalized (normalize_dinov2) <- what restrav.sh trains with + +Label convention (official): REAL=1, FAKE=0. AUROC is positive-class-invariant; AP is fake-positive. +""" +import os, sys, glob, re, random, time +os.environ.setdefault("HF_HOME", "/hddstore/ssalehi/SynthScope/.cache/huggingface") +os.environ.setdefault("TORCH_HOME", "/hddstore/ssalehi/SynthScope/.cache/torch") +import numpy as np, torch, torch.nn as nn, torch.nn.functional as F +import torchvision.transforms.functional as TF +from torchvision.transforms import InterpolationMode +import decord +from sklearn.model_selection import train_test_split +from sklearn.metrics import roc_auc_score, average_precision_score + +GV = "/hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val" +EXTRACTED = "/hddstore/ssalehi/SynthScope/d3_repro/extracted" +CACHE = "/hddstore/ssalehi/SynthScope/restrav_xgen_cache" +os.makedirs(CACHE, exist_ok=True) +DEV = "cuda" +T, WIN = 24, 2.0 +K_FAKE_PER_GEN = int(os.environ.get("K_FAKE_PER_GEN", "200")) +N_REAL_GV = int(os.environ.get("N_REAL_GV", "2000")) +N_EX = int(os.environ.get("N_EX", "1500")) # per class for the extracted (cross-dataset) train pool + +IMNET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(DEV) +IMNET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(DEV) + +print("loading DINOv2 ViT-S/14...", flush=True) +dinov2 = torch.hub.load("facebookresearch/dinov2", "dinov2_vits14").eval().to(DEV) + + +def decode_clip(path): + vr = decord.VideoReader(path, ctx=decord.cpu(0)) + n = len(vr); fps = float(vr.get_avg_fps()) or 30.0; dur = n / fps + center = dur / 2.0; start = max(0.0, center - WIN / 2) + delta = WIN / max(T - 1, 1) + idxs = [min(int(round((start + i * delta) * fps)), n - 1) for i in range(T)] + frames = vr.get_batch(idxs).asnumpy() + return torch.from_numpy(frames).permute(0, 3, 1, 2) + + +def preprocess(frames): + frames = frames.float() / 255.0 + return torch.stack([TF.resize(x, [224, 224], InterpolationMode.BICUBIC, antialias=True) for x in frames]) + + +def geom21(Z): + dz = Z[:, 1:] - Z[:, :-1]; d = dz.norm(dim=-1) + cos = F.cosine_similarity(dz[:, :-1], dz[:, 1:], dim=-1) + th = torch.rad2deg(torch.acos(cos.clamp(-1, 1))) + def m4(x): return [x.mean(-1), x.amin(-1), x.amax(-1), x.var(-1, unbiased=False)] + stats = torch.stack(m4(d) + m4(th), dim=1) + return torch.cat([d[:, :7], th[:, :6], stats], dim=1)[0].cpu().numpy() + + +@torch.no_grad() +def feats_both(path): + clip = preprocess(decode_clip(path)).to(DEV) + + def run(x): + f = dinov2.forward_features(x) + tokens = torch.cat([f["x_norm_clstoken"].unsqueeze(1), f["x_norm_patchtokens"]], dim=1) + return geom21(tokens.flatten(1).unsqueeze(0)) + raw = run(clip) + norm = run((clip - IMNET_MEAN) / IMNET_STD) + return raw, norm + + +def build(items, tag): + """items: list of (path, label, gen). Returns Xraw, Xnorm, y, gens (np arrays), caching to npz.""" + cache = os.path.join(CACHE, f"feat_{tag}.npz") + if os.path.exists(cache): + z = np.load(cache, allow_pickle=True) + print(f"[{tag}] loaded {len(z['y'])} cached features", flush=True) + return z["Xraw"], z["Xnorm"], z["y"], z["gens"] + Xr, Xn, y, gens = [], [], [], [] + t0 = time.time() + for i, (p, lab, g) in enumerate(items): + try: + r, n = feats_both(p) + if np.isfinite(r).all() and np.isfinite(n).all(): + Xr.append(r); Xn.append(n); y.append(lab); gens.append(g) + except Exception: + pass + if (i + 1) % 200 == 0: + print(f" [{tag}] {i+1}/{len(items)} ({(time.time()-t0)/(i+1):.2f}s/vid)", flush=True) + Xr = np.array(Xr, np.float32); Xn = np.array(Xn, np.float32) + y = np.array(y, np.int64); gens = np.array(gens) + np.savez(cache, Xraw=Xr, Xnorm=Xn, y=y, gens=gens) + print(f"[{tag}] extracted+cached {len(y)} features", flush=True) + return Xr, Xn, y, gens + + +def train_eval(Xtr, ytr, Xte, yte, seed=0): + """Official recipe. Label real=1. Returns (AUROC, AP_fake). Averaged over a few seeds outside.""" + torch.manual_seed(seed); np.random.seed(seed) + mu = Xtr.mean(0, keepdims=True); sd = Xtr.std(0, keepdims=True) + 1e-8 + Xtr = (Xtr - mu) / sd; Xte = (Xte - mu) / sd + net = nn.Sequential(nn.Linear(21, 64), nn.ReLU(), nn.Linear(64, 32), nn.ReLU(), nn.Linear(32, 1)).to(DEV) + opt = torch.optim.Adam(net.parameters(), lr=1e-3); crit = nn.BCEWithLogitsLoss() + Xt = torch.tensor(Xtr, device=DEV); yt = torch.tensor(ytr, dtype=torch.float32, device=DEV).unsqueeze(1) + for ep in range(10): + perm = torch.randperm(len(Xt), device=DEV) + for i in range(0, len(Xt), 32): + b = perm[i:i+32]; opt.zero_grad() + loss = crit(net(Xt[b]), yt[b]); loss.backward(); opt.step() + net.eval() + with torch.no_grad(): + p_real = torch.sigmoid(net(torch.tensor(Xte, device=DEV))).cpu().numpy().ravel() + auc = roc_auc_score(yte, p_real) + ap_fake = average_precision_score(1 - yte, 1 - p_real) + return auc, ap_fake + + +def mean_eval(Xtr, ytr, Xte, yte, seeds=(0, 1, 2)): + rs = [train_eval(Xtr, ytr, Xte, yte, s) for s in seeds] + auc = np.mean([r[0] for r in rs]); ap = np.mean([r[1] for r in rs]) + return auc, ap + + +def balance(X, y, rng): + """Subsample to equal priors (official train.py).""" + ir = np.where(y == 1)[0]; ifk = np.where(y == 0)[0] + k = min(len(ir), len(ifk)) + sel = np.concatenate([rng.choice(ir, k, replace=False), rng.choice(ifk, k, replace=False)]) + rng.shuffle(sel) + return X[sel], y[sel] + + +def rperm_split(perm): + h = len(perm) // 2 + return perm[:h], perm[h:] + + +def main(): + random.seed(0) + # ---- assemble GenVideo-Val item list (real=1, fake=0), tagged by generator ---- + gvr = sorted(glob.glob(GV + "/Real/*.mp4")); random.Random(0).shuffle(gvr) + gv_items = [(p, 1, "real") for p in gvr[:N_REAL_GV]] + for gdir in sorted(glob.glob(GV + "/Fake/*/")): + g = os.path.basename(gdir.rstrip("/")) + fk = sorted(glob.glob(gdir + "*.mp4")); random.Random(hash(g) % 100).shuffle(fk) + gv_items += [(p, 0, g) for p in fk[:K_FAKE_PER_GEN]] + print(f"GenVideo-Val items: {len(gv_items)}", flush=True) + Xr_gv, Xn_gv, y_gv, g_gv = build(gv_items, "genvideo") + + # ---- extracted (VidProM/DeMamba) pool for cross-dataset ---- + ex = sorted(glob.glob(EXTRACTED + "/*.mp4")) + exr = [p for p in ex if re.search(r"_real_", os.path.basename(p))] + exf = [p for p in ex if not re.search(r"_real_", os.path.basename(p))] + random.Random(2).shuffle(exr); random.Random(3).shuffle(exf) + ex_items = [(p, 1, "real") for p in exr[:N_EX]] + [(p, 0, "vidprom") for p in exf[:N_EX]] + print(f"extracted items: {len(ex_items)}", flush=True) + Xr_ex, Xn_ex, y_ex, g_ex = build(ex_items, "extracted") + + for variant, Xgv, Xex in [("raw[0,1] (OFFICIAL)", Xr_gv, Xr_ex), ("imagenet-norm (restrav.sh)", Xn_gv, Xn_ex)]: + print(f"\n{'='*78}\n### FEATURE VARIANT: {variant}\n{'='*78}", flush=True) + + # (A) IN-DISTRIBUTION + rng = np.random.default_rng(42) + Xb, yb = balance(Xgv, y_gv, rng) + Xtr, Xte, ytr, yte = train_test_split(Xb, yb, test_size=0.5, stratify=yb, random_state=42) + auc, ap = mean_eval(Xtr, ytr, Xte, yte) + print(f"(A) IN-DISTRIBUTION (official 50/50, shared gens): AUROC={auc:.3f} AP(fake)={ap:.3f} n_test={len(yte)}", flush=True) + + # (B) UNSEEN-GENERATOR (leave-one-generator-out, real source fixed) + print("(B) UNSEEN-GENERATOR (paper Table-3 style, leave-one-gen-out, SAME real source):", flush=True) + real_idx = np.where(y_gv == 1)[0] + rng2 = np.random.default_rng(7); rperm = rng2.permutation(real_idx) + tr_real, te_real = rperm_split(rperm) + gens = sorted(set(g_gv[y_gv == 0])) + per = [] + for held in gens: + tr_f = np.where((y_gv == 0) & (g_gv != held))[0] + te_f = np.where((y_gv == 0) & (g_gv == held))[0] + Xtr = np.concatenate([Xgv[tr_real], Xgv[tr_f]]); ytr = np.concatenate([np.ones(len(tr_real)), np.zeros(len(tr_f))]).astype(np.int64) + Xte = np.concatenate([Xgv[te_real], Xgv[te_f]]); yte = np.concatenate([np.ones(len(te_real)), np.zeros(len(te_f))]).astype(np.int64) + rngb = np.random.default_rng(1); Xtr, ytr = balance(Xtr, ytr, rngb) + auc, ap = mean_eval(Xtr, ytr, Xte, yte) + per.append((held, auc, ap, len(te_f))) + print(f" held-out={held:<12} AUROC={auc:.3f} AP(fake)={ap:.3f} (n_fake_test={len(te_f)})", flush=True) + print(f" >>> MEAN over unseen generators: AUROC={np.mean([p[1] for p in per]):.3f} AP(fake)={np.mean([p[2] for p in per]):.3f}", flush=True) + + # (C) CROSS-DATASET (our README "cross-generator") + rngc = np.random.default_rng(5); Xtr, ytr = balance(Xex, y_ex, rngc) + auc, ap = mean_eval(Xtr, ytr, Xgv, y_gv) + print(f"(C) CROSS-DATASET (train=extracted/VidProM -> test=GenVideo-Val, real source SHIFTS): AUROC={auc:.3f} AP(fake)={ap:.3f} n_test={len(y_gv)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/retrain_restrav.py b/interface/scripts/retrain_restrav.py new file mode 100644 index 0000000000000000000000000000000000000000..098f114740811232dfeee851850d1b63666acc29 --- /dev/null +++ b/interface/scripts/retrain_restrav.py @@ -0,0 +1,133 @@ +"""Retrain the deployed ReStraV checkpoint with the corrected (official) recipe. + +The previous restrav.pth was trained on ImageNet-normalized frames but is *served* raw [0,1] +(registry video_norm = identity) -- a train/serve mismatch that collapsed it (AUROC ~0.50, +real_acc ~2%). This retrains the SynthScope `SynthScopeReStraVModel` head end-to-end on raw [0,1] frames, +so training and serving use the exact same DINOv2 features. + +Key fidelity choices: + * Features come from the model's OWN ``_encode`` + ``_geometry_features`` (so they are + bit-identical to what the deployed forward computes, incl. the official [mean,min,max,var] + summary order), with frames decoded by the SAME serving pipeline (media.video_clip_tensor, + center window + BICUBIC, raw [0,1]). + * Official training recipe: standardize (BatchNorm1d running stats, train mode), MLP + 21->64->32->1, BCEWithLogits, plain Adam lr 1e-3 (no weight decay), 10 epochs, batch 32. + * Save the FINAL-epoch state_dict (no val-loss checkpoint selection). + +Run: conda run -n SynthScope python interface/scripts/retrain_restrav.py [--n 3000] +""" +import argparse +import glob +import random +import sys +import time +from pathlib import Path + +import torch +import torch.nn as nn + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # interface/ +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # repo root (config_utils) +from backend.registry import get_spec # noqa: E402 +from backend import media # noqa: E402 +from backend.engine import _build_model # noqa: E402 + +DEV = "cuda" if torch.cuda.is_available() else "cpu" +TRAIN_ROOT = "/hddstore/ssalehi/SynthScope/restrav_data/train/genvideo" +OUT_DIR = Path("/hddstore/ssalehi/SynthScope/checkpoints/_train/restrav_fixed") +DEPLOY_LINK = Path("/hddstore/ssalehi/SynthScope/checkpoints/restrav.pth") + + +def build_model(spec): + from config_utils import compose_config + cfg = compose_config(spec.overrides) + model = _build_model(cfg).to(DEV).float() + model.eval() # frozen backbone stays in eval + return model + + +@torch.no_grad() +def extract_features(model, spec, items, tag): + X, y = [], [] + t0 = time.time() + for i, (p, lab) in enumerate(items): + try: + clip = media.video_clip_tensor( + p, spec.num_frames, spec.fps, spec.frame_size, + window_center=spec.window_center, interp=spec.resize_interp, + ) # [T,3,H,W] raw [0,1] + z = model._encode(clip.unsqueeze(0).to(DEV)) # [1,T,D] + feat = model._geometry_features(z)[0] # [21] + if torch.isfinite(feat).all(): + X.append(feat.cpu()); y.append(lab) + except Exception as e: + if i < 5: + print(f" skip {Path(p).name}: {e}", flush=True) + if (i + 1) % 300 == 0: + print(f" [{tag}] {i+1}/{len(items)} ({(time.time()-t0)/(i+1):.2f}s/vid)", flush=True) + return torch.stack(X), torch.tensor(y, dtype=torch.float32) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--n", type=int, default=3000, help="videos per class") + ap.add_argument("--epochs", type=int, default=10) + args = ap.parse_args() + + spec = get_spec("restrav") + model = build_model(spec) + print(f"model built on {DEV}; serving recipe: num_frames={spec.num_frames} " + f"window_center={spec.window_center} interp={spec.resize_interp} video_norm={spec.video_norm}", flush=True) + + reals = sorted(glob.glob(TRAIN_ROOT + "/0_real/*.mp4")) + fakes = sorted(glob.glob(TRAIN_ROOT + "/1_fake/*.mp4")) + random.Random(0).shuffle(reals); random.Random(1).shuffle(fakes) + # SynthScope label convention: 0 = real, 1 = fake. + items = [(p, 0) for p in reals[:args.n]] + [(p, 1) for p in fakes[:args.n]] + random.Random(2).shuffle(items) + print(f"extracting features for {len(items)} videos (raw [0,1], model._encode)...", flush=True) + X, y = extract_features(model, spec, items, "train") + X = X.to(DEV); y = y.to(DEV) + print(f"features: {tuple(X.shape)} reals={(y==0).sum().item()} fakes={(y==1).sum().item()}", flush=True) + + # --- train head (official recipe). feature_norm is BatchNorm1d(affine=False): no params, + # just running stats accumulated in train mode. classifier holds the trainable weights. --- + model.feature_norm.train(); model.classifier.train() + opt = torch.optim.Adam(model.classifier.parameters(), lr=1e-3) # wd=0 (plain Adam) + crit = nn.BCEWithLogitsLoss() + for ep in range(args.epochs): + perm = torch.randperm(len(X), device=DEV) + tot = 0.0 + for i in range(0, len(X), 32): + b = perm[i:i+32] + opt.zero_grad() + logit = model.classifier(model.feature_norm(X[b])).squeeze(-1) # fake logit + loss = crit(logit, y[b]) + loss.backward(); opt.step() + tot += loss.item() * len(b) + print(f" epoch {ep+1}/{args.epochs} loss={tot/len(X):.4f}", flush=True) + + # --- save FINAL state_dict (matches what engine._load_checkpoint expects) --- + model.eval() + OUT_DIR.mkdir(parents=True, exist_ok=True) + out = OUT_DIR / "model_final.pth" + torch.save(model.state_dict(), out) + print(f"saved {out}", flush=True) + + # quick in-sample sanity (real_acc / fake_acc on the train features) + with torch.no_grad(): + p_fake = torch.sigmoid(model.classifier(model.feature_norm(X)).squeeze(-1)) + pred = (p_fake > 0.5).float() + racc = (pred[y == 0] == 0).float().mean().item() + facc = (pred[y == 1] == 1).float().mean().item() + print(f"in-sample: real_acc={racc*100:.1f} fake_acc={facc*100:.1f}", flush=True) + + # relink the deployed checkpoint + if DEPLOY_LINK.is_symlink() or DEPLOY_LINK.exists(): + DEPLOY_LINK.unlink() + DEPLOY_LINK.symlink_to(out) + print(f"relinked {DEPLOY_LINK} -> {out}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/interface/scripts/run_all_training.sh b/interface/scripts/run_all_training.sh new file mode 100644 index 0000000000000000000000000000000000000000..8d0339f6f23932958ab6600c44564511d1edd2b5 --- /dev/null +++ b/interface/scripts/run_all_training.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Old AGID_* variables still work: map each to its SYNTHSCOPE_* name unless that is already set. +for _v in $(compgen -e | grep '^AGID_' || true); do _n="SYNTHSCOPE_${_v#AGID_}"; [ -n "${!_n:-}" ] || export "$_n=${!_v}"; done +# Unattended orchestrator: finishes the whole "train the missing detectors" pipeline on cn023 +# so it completes even if the user disconnects. It does NOT relaunch the jobs already running +# (clip/dinov2/vib_net probes, restrav) -- it waits for them, then trains the rest, promoting +# and verifying each, calibrates D3, and rewrites the manifest. Detach with: +# +# setsid bash interface/scripts/run_all_training.sh >/dev/null 2>&1 & +# +# Progress: tail -f $CKPT_DIR/_train/orchestrator.log +set +e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" +CONDA="${CONDA_EXE:-/home/ssalehi/miniconda3/bin/conda}" +CKPT_DIR="${SYNTHSCOPE_CKPT_DIR:-/hddstore/ssalehi/SynthScope/checkpoints}" +ORCH_LOG="$CKPT_DIR/_train/orchestrator.log" +mkdir -p "$CKPT_DIR/_train" +TRAIN=interface/scripts/train_model.sh + +log() { echo "[$(date '+%m-%d %H:%M:%S')] $*" | tee -a "$ORCH_LOG"; } +running() { pgrep -af "main.py" | grep -q "train_$1\b"; } +wait_for() { # wait until none of the named jobs are running + local names=("$@") + while :; do + local any=0 + for n in "${names[@]}"; do running "$n" && any=1; done + [ "$any" -eq 0 ] && break + sleep 30 + done +} +promote() { "$CONDA" run --no-capture-output -n SynthScope python interface/scripts/promote_checkpoint.py "$1" 2>&1 | tee -a "$ORCH_LOG"; } +verify() { "$CONDA" run --no-capture-output -n SynthScope python interface/scripts/verify_model.py "$1" --n 240 2>&1 | grep -v libpng | tee -a "$ORCH_LOG"; } +collect() { "$CONDA" run --no-capture-output -n SynthScope python interface/scripts/collect_checkpoints.py 2>&1 | tee -a "$ORCH_LOG"; } +have_ckpt() { ls "$CKPT_DIR/_train/$1"/model_*.pth >/dev/null 2>&1; } +finish() { # promote+verify if a checkpoint was produced + if have_ckpt "$1"; then log "promote+verify $1"; promote "$1"; verify "$1" + else log "WARN: $1 produced no checkpoint (see _train/$1/train.log)"; fi +} + +log "=== orchestrator start ===" + +# Phase 1: the already-running probes. +log "waiting for probes: clip dinov2 vib_net ..." +wait_for clip dinov2 vib_net +sleep 15 +for m in clip dinov2 vib_net; do finish "$m"; done + +# Phase 2: heavy wave A on the freed GPUs 0-5 (restrav keeps 6,7 until done). +log "launching wave A: adapted_dino(0,1) cnn(2,3) safe(4,5)" +setsid bash "$TRAIN" adapted_dino 0,1 29561 2 >/dev/null 2>&1 & +setsid bash "$TRAIN" cnn 2,3 29562 3 >/dev/null 2>&1 & +setsid bash "$TRAIN" safe 4,5 29563 10 >/dev/null 2>&1 & +sleep 20 +wait_for adapted_dino cnn safe +sleep 15 +for m in adapted_dino cnn safe; do finish "$m"; done + +# Phase 3: heavy wave B on GPUs 0-3. +log "launching wave B: spai(0,1) co_spy(2,3)" +setsid bash "$TRAIN" spai 0,1 29564 5 >/dev/null 2>&1 & +setsid bash "$TRAIN" co_spy 2,3 29565 3 >/dev/null 2>&1 & +sleep 20 +wait_for spai co_spy +sleep 15 +for m in spai co_spy; do finish "$m"; done + +# Phase 4: restrav (started earlier on 6,7). +log "waiting for restrav ..." +wait_for restrav +sleep 10 +if have_ckpt restrav; then + log "promote restrav + full GenVideo-Val eval" + promote restrav + "$CONDA" run --no-capture-output -n SynthScope python interface/scripts/eval_restrav_full.py --n 600 2>&1 | grep -v libpng | tee -a "$ORCH_LOG" +else + log "WARN: restrav produced no checkpoint" +fi + +# Phase 5: D3 calibration (GPUs free now). +log "calibrating D3 ..." +CUDA_VISIBLE_DEVICES=0 "$CONDA" run --no-capture-output -n SynthScope python interface/scripts/calibrate_d3.py --n 200 2>&1 | grep -v libpng | tee -a "$ORCH_LOG" + +# Phase 6: refresh manifest + summary. +log "refreshing manifest ..." +collect +log "=== orchestrator done ===" +log "checkpoints present:" +ls -lL "$CKPT_DIR"/*.pth 2>/dev/null | awk '{print " "$NF}' | tee -a "$ORCH_LOG" +cat "$CKPT_DIR/MANIFEST.md" 2>/dev/null | tee -a "$ORCH_LOG" diff --git a/interface/scripts/train_model.sh b/interface/scripts/train_model.sh new file mode 100644 index 0000000000000000000000000000000000000000..957e71febff3f46ffe49115dd64235ac2d36bd04 --- /dev/null +++ b/interface/scripts/train_model.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# Train one SynthScope detector with checkpoint-saving enabled, into the central checkpoint dir's +# staging area. Recipes mirror experiments/image/*.sh; epochs are reduced for turnaround and +# overridable. After training, run promote_checkpoint.py to publish .pth. +# +# bash interface/scripts/train_model.sh [epochs] +# e.g. bash interface/scripts/train_model.sh clip 0,1 29510 2 +# +# Notes: +# * save_policy is forced to "min" (lowest val loss) because the trainer initialises +# best_val_loss=+inf, so the repo default save_policy=max never writes a checkpoint. +# * Runs in the SynthScope conda env; logs to $CKPT_DIR/_train//train.log. +set -uo pipefail +# Old AGID_* variables still work: map each to its SYNTHSCOPE_* name unless that is already set. +for _v in $(compgen -e | grep '^AGID_' || true); do _n="SYNTHSCOPE_${_v#AGID_}"; [ -n "${!_n:-}" ] || export "$_n=${!_v}"; done + +MODEL="${1:?usage: train_model.sh [epochs]}" +GPUS="${2:?need GPU list, e.g. 0,1}" +PORT="${3:?need master port, e.g. 29510}" +EPOCHS_OVERRIDE="${4:-}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" +CONDA="${CONDA_EXE:-/home/ssalehi/miniconda3/bin/conda}" +CKPT_DIR="${SYNTHSCOPE_CKPT_DIR:-/hddstore/ssalehi/SynthScope/checkpoints}" +CACHE="${SYNTHSCOPE_CACHE_DIR:-/hddstore/ssalehi/SynthScope/.cache}" +SAVE_DIR="$CKPT_DIR/_train/$MODEL" +mkdir -p "$SAVE_DIR" + +export HF_HOME="$CACHE/huggingface" +export TORCH_HOME="$CACHE/torch" +export TRITON_CACHE_DIR="$CACHE/triton" +export OMP_NUM_THREADS=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export CUDA_DEVICE_ORDER=PCI_BUS_ID +export DINO_V3_REPO_DIR="${DINO_V3_REPO_DIR:-/home/ssalehi/.cache/torch/hub/facebookresearch_dinov3}" +export DINO_V3_WEIGHTS_DIR="${DINO_V3_WEIGHTS_DIR:-/hddstore/ssalehi/SynthScope/dinov3_checkpoints}" +export CUDA_VISIBLE_DEVICES="$GPUS" +NPROC=$(awk -F',' '{print NF}' <<< "$GPUS") + +# Common transform lists. +T_CLIP_TRAIN="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" +T_CLIP_EVAL="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" +T_IMNET_TRAIN="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" +T_IMNET_EVAL="[resize_short_256,center_crop_224,to_dtype,normalize_dinov2]" +T_RAW_TRAIN="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype]" +T_RAW_EVAL="[resize_short_256,center_crop_224,to_dtype]" + +OV=() # Hydra overrides +case "$MODEL" in + clip) + OV=(ModelConfig.model_type=clip ModelConfig.backbone_name=CLIP:ViT-L/14 + ModelConfig.use_adapter=false ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true + dataset@DatasetConfig=image/cnndetection + DatasetConfig.train_transform_list=$T_CLIP_TRAIN DatasetConfig.eval_transform_list=$T_CLIP_EVAL) + EPOCHS=${EPOCHS_OVERRIDE:-2} ;; + dinov2) + OV=(ModelConfig.model_type=dinov2 ModelConfig.backbone_name=dinov2_vitl14 + ModelConfig.use_adapter=false ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true + dataset@DatasetConfig=image/cnndetection + DatasetConfig.train_transform_list=$T_IMNET_TRAIN DatasetConfig.eval_transform_list=$T_IMNET_EVAL) + EPOCHS=${EPOCHS_OVERRIDE:-2} ;; + dinov3) + # Frozen DINOv3 ViT-L/16 probe (weights at $DINO_V3_WEIGHTS_DIR). DINOv3 normalization + 256 crops. + OV=(ModelConfig.model_type=dinov3 ModelConfig.backbone_name=dinov3_vitl16 + ModelConfig.use_adapter=false ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true + dataset@DatasetConfig=image/cnndetection + DatasetConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype,normalize_dinov3]" + DatasetConfig.eval_transform_list="[resize_short_256,center_crop_256,to_dtype,normalize_dinov3]") + EPOCHS=${EPOCHS_OVERRIDE:-2} ;; + pgc) + # LoRA-DINOv2-Large + residual stream. Train on the ~120K ProGAN subset for turnaround; + # crop-only (no resize) preserves the quantization residual; normalize_dinov2 = ImageNet. + OV=(trainer@TrainerConfig=image/pgc model@ModelConfig=image/pgc + dataset@DatasetConfig=image/cospy_subset + DatasetConfig.train_batch_size=32 DatasetConfig.val_batch_size=64 DatasetConfig.test_batch_size=64 + DatasetConfig.train_transform_list="[random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" + DatasetConfig.eval_transform_list="[center_crop_224,to_dtype,normalize_dinov2]") + EPOCHS=${EPOCHS_OVERRIDE:-2} ;; + adapted_dino) + # Adapters at every block retain backbone activations for backprop -> small batch. + OV=(ModelConfig.model_type=adapted_dino ModelConfig.backbone_name=dinov2_vitl14 + ModelConfig.use_adapter=true ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true + dataset@DatasetConfig=image/cnndetection + DatasetConfig.train_batch_size=64 DatasetConfig.val_batch_size=256 DatasetConfig.test_batch_size=256 + DatasetConfig.train_transform_list=$T_IMNET_TRAIN DatasetConfig.eval_transform_list=$T_IMNET_EVAL) + EPOCHS=${EPOCHS_OVERRIDE:-2} ;; + cnn) + # Full ResNet-50 fine-tune (backbone trains) -> moderate batch. + OV=(ModelConfig.model_type=cnn ModelConfig.freeze_backbone=false + dataset@DatasetConfig=image/cnndetection + DatasetConfig.train_batch_size=128 DatasetConfig.val_batch_size=512 DatasetConfig.test_batch_size=512 + DatasetConfig.train_transform_list=$T_IMNET_TRAIN DatasetConfig.eval_transform_list=$T_IMNET_EVAL) + EPOCHS=${EPOCHS_OVERRIDE:-3} ;; + vib_net) + OV=(trainer@TrainerConfig=image/vib_net model@ModelConfig=image/vib_net + dataset@DatasetConfig=image/genimage + DatasetConfig.train_transform_list=$T_CLIP_TRAIN DatasetConfig.eval_transform_list=$T_CLIP_EVAL) + EPOCHS=${EPOCHS_OVERRIDE:-3} ;; + spai) + OV=(trainer@TrainerConfig=image/spai model@ModelConfig=image/spai + dataset@DatasetConfig=image/spai_ldm + DatasetConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" + DatasetConfig.eval_transform_list="[resize_short_448,center_crop_448,to_dtype]") # 448 raw [0,1] -> 2x2 grid + SCA + EPOCHS=${EPOCHS_OVERRIDE:-5} ;; + co_spy) + OV=(trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy + dataset@DatasetConfig=image/cospy_subset + DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" + DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]") + EPOCHS=${EPOCHS_OVERRIDE:-3} ;; + safe) + # AIGIBench (the paper recipe) is not on this node; train SAFE from scratch on ProGAN instead. + OV=(trainer@TrainerConfig=image/safe ModelConfig.model_type=safe ModelConfig.freeze_backbone=false + dataset@DatasetConfig=image/cnndetection + DatasetConfig.train_transform_list="[random_crop_256_pad,random_horizontal_flip,random_rotation_180,color_jitter_safe,to_dtype,random_mask_safe]" + DatasetConfig.eval_transform_list="[center_crop_256,to_dtype]") + EPOCHS=${EPOCHS_OVERRIDE:-10} ;; + restrav) + OV=(trainer@TrainerConfig=video/restrav model@ModelConfig=video/restrav + dataset@DatasetConfig=video/genvideo DatasetConfig.root_dir=/hddstore/ssalehi/SynthScope/restrav_data + DatasetConfig.num_frames=24 DatasetConfig.num_workers=8 + DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=16 DatasetConfig.test_batch_size=16 + ++TrainerConfig.test_interval=5 ++TrainerConfig.weight_decay=0.0 ++TrainerConfig.warmup.mode=off + ++DatasetConfig.window_center=true ++DatasetConfig.resize_interp=bicubic + DatasetConfig.train_transform_list="[]" DatasetConfig.eval_transform_list="[]") # raw [0,1], center window + BICUBIC (official ReStraV) + EPOCHS=${EPOCHS_OVERRIDE:-10} ;; + *) echo "unknown model: $MODEL"; exit 2 ;; +esac + +LOG="$SAVE_DIR/train.log" +echo "=== train $MODEL on GPUs=$GPUS ($NPROC proc) port=$PORT epochs=$EPOCHS ===" | tee "$LOG" +echo "save_path=$SAVE_DIR" | tee -a "$LOG" + +# CUDA init can flake on this node's old kernel; retry the pre-check before launching. +for try in 1 2 3 4 5; do + if "$CONDA" run -n SynthScope python -c "import torch; assert torch.cuda.is_available(); torch.zeros(1,device='cuda')" 2>/dev/null; then + break + fi + echo "[precheck] CUDA not ready (try $try), retrying..." | tee -a "$LOG"; sleep 5 +done + +"$CONDA" run --no-capture-output -n SynthScope accelerate launch \ + --config_file accelerate_config.yaml --num_processes "$NPROC" --main_process_port "$PORT" \ + main.py "${OV[@]}" \ + ++TrainerConfig.epochs="$EPOCHS" \ + ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 \ + ++TrainerConfig.save_path="$SAVE_DIR" \ + metrics=[real_accuracy,fake_accuracy,AP,AUC] \ + logger.mode=offline logger.name="train_$MODEL" \ + 2>&1 | grep --line-buffered -v -E "libpng (error|warning)" | tee -a "$LOG" + +echo "=== done $MODEL (exit ${PIPESTATUS[0]}) ===" | tee -a "$LOG" diff --git a/interface/scripts/verify_model.py b/interface/scripts/verify_model.py new file mode 100644 index 0000000000000000000000000000000000000000..0ecca9b85c3463379b877edd7cf80adfc73f4f41 --- /dev/null +++ b/interface/scripts/verify_model.py @@ -0,0 +1,90 @@ +"""Verify a (trained or official) image detector by scoring a balanced real/fake sample +from the local ForenSynths/CNNDetection test set, through the SAME engine the web app uses. + +Reports accuracy@0.5, real/fake accuracy, AP and AUC so the checkpoint's quality is a real, +reproducible number (not a training-log artifact). + + conda run -n SynthScope python interface/scripts/verify_model.py [--n 200] [--gens progan,biggan] +""" +import argparse +import glob +import os +import random +import sys +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from backend.engine import get_engine # noqa: E402 +from backend.registry import get_spec # noqa: E402 + +TEST_ROOT = "/hddstore/ssalehi/SynthScope/CNNDetection/test" + + +def list_split(gen, sub): + d = os.path.join(TEST_ROOT, gen, sub) + fs = [] + for e in ("*.png", "*.jpg", "*.jpeg", "*.JPEG", "*.webp", "*.bmp"): + fs += glob.glob(os.path.join(d, "**", e), recursive=True) + return sorted(set(fs)) + + +def ap_auc(y, p): + y = np.asarray(y); p = np.asarray(p) + order = np.argsort(-p, kind="mergesort"); y = y[order] + tp = np.cumsum(y == 1); fp = np.cumsum(y == 0) + npos, nneg = (y == 1).sum(), (y == 0).sum() + if npos == 0 or nneg == 0: + return float("nan"), float("nan") + prec = tp / np.maximum(tp + fp, 1); rec = tp / npos + rec = np.concatenate([[0.0], rec]); prec = np.concatenate([[1.0], prec]) + ap = float(np.sum((rec[1:] - rec[:-1]) * prec[1:])) + tpr = np.concatenate([[0.0], tp / npos]); fpr = np.concatenate([[0.0], fp / nneg]) + auc = float(np.sum((fpr[1:] - fpr[:-1]) * (tpr[1:] + tpr[:-1]) / 2.0)) + return ap, auc + + +@torch.no_grad() +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("key") + ap.add_argument("--n", type=int, default=200, help="images per class") + ap.add_argument("--gens", default="progan,biggan,stylegan,cyclegan") + ap.add_argument("--bs", type=int, default=64) + args = ap.parse_args() + + engine = get_engine() + spec = get_spec(args.key) + model = engine._get_model(spec) + + items = [] + for gen in args.gens.split(","): + reals = list_split(gen, "0_real"); fakes = list_split(gen, "1_fake") + random.Random(0).shuffle(reals); random.Random(1).shuffle(fakes) + per = max(1, args.n // len(args.gens.split(","))) + items += [(p, 0) for p in reals[:per]] + [(p, 1) for p in fakes[:per]] + random.Random(2).shuffle(items) + print(f"{args.key}: scoring {len(items)} images from [{args.gens}] on {engine.device}") + + probs, labels = [], [] + for i in range(0, len(items), args.bs): + chunk = items[i:i + args.bs] + batch = torch.stack([spec.transform(Image.open(p).convert("RGB")) for p, _ in chunk]).to(engine.device) + probs += engine._fake_prob(model, spec, batch).float().cpu().tolist() + labels += [l for _, l in chunk] + + probs = np.array(probs); labels = np.array(labels) + pred = (probs > 0.5).astype(int) + acc = float((pred == labels).mean()) + racc = float((pred[labels == 0] == 0).mean()) + facc = float((pred[labels == 1] == 1).mean()) + apv, aucv = ap_auc(labels, probs) + print(f" acc@0.5={acc*100:.2f} real_acc={racc*100:.2f} fake_acc={facc*100:.2f} " + f"AP={apv*100:.2f} AUC={aucv*100:.2f} (mean P(fake)={probs.mean():.3f})") + + +if __name__ == "__main__": + main() diff --git a/interface/static/index.html b/interface/static/index.html new file mode 100644 index 0000000000000000000000000000000000000000..d3ec7ae054d2c4db9325bc1192ad993f021aa887 --- /dev/null +++ b/interface/static/index.html @@ -0,0 +1,1280 @@ + + + + + + SynthScope — AI-Generated Media Detector + + + + + + + +
+ + + diff --git a/interface/static/vendor/htm.umd.js b/interface/static/vendor/htm.umd.js new file mode 100644 index 0000000000000000000000000000000000000000..81bc89f71e2990645711bd247e71b44ddc60271f --- /dev/null +++ b/interface/static/vendor/htm.umd.js @@ -0,0 +1 @@ +!function(n,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):n.htm=e()}(this,function(){var n=function(e,t,u,s){var r;t[0]=0;for(var p=1;p=5&&((s||!n&&5===u)&&(p.push(u,0,s,t),u=6),n&&(p.push(u,n,0,t),u=6)),s=""},o=0;o"===e?(u=1,s=""):s=e+s[0]:r?e===r?r="":s+=e:'"'===e||"'"===e?r=e:">"===e?(h(),u=1):u&&("="===e?(u=5,t=s,s=""):"/"===e&&(u<5||">"===n[o][f+1])?(h(),3===u&&(p=p[0]),u=p,(p=p[0]).push(2,0,u),u=0):" "===e||"\t"===e||"\n"===e||"\r"===e?(h(),u=2):s+=e),3===u&&"!--"===s&&(u=4,p=p[0])}return h(),p}(t)),u),arguments,[])).length>1?u:u[0]}}); diff --git a/interface/static/vendor/react-dom.production.min.js b/interface/static/vendor/react-dom.production.min.js new file mode 100644 index 0000000000000000000000000000000000000000..e8a0213d7e1b08f28dff8f09c63b81bc080e944c --- /dev/null +++ b/interface/static/vendor/react-dom.production.min.js @@ -0,0 +1,267 @@ +/** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +(function(){/* + Modernizr 3.0.0pre (Custom Build) | MIT +*/ +'use strict';(function(Q,mb){"object"===typeof exports&&"undefined"!==typeof module?mb(exports,require("react")):"function"===typeof define&&define.amd?define(["exports","react"],mb):(Q=Q||self,mb(Q.ReactDOM={},Q.React))})(this,function(Q,mb){function n(a){for(var b="https://reactjs.org/docs/error-decoder.html?invariant="+a,c=1;cb}return!1}function Y(a,b,c,d,e,f,g){this.acceptsBooleans=2===b||3===b||4===b;this.attributeName=d;this.attributeNamespace=e;this.mustUseProperty=c;this.propertyName=a;this.type=b;this.sanitizeURL=f;this.removeEmptyString=g}function $d(a,b,c,d){var e=R.hasOwnProperty(b)?R[b]:null;if(null!==e?0!==e.type:d||!(2h||e[g]!==f[h]){var k="\n"+e[g].replace(" at new "," at ");a.displayName&&k.includes("")&&(k=k.replace("",a.displayName));return k}while(1<=g&&0<=h)}break}}}finally{ce=!1,Error.prepareStackTrace=c}return(a=a?a.displayName||a.name:"")?bc(a): +""}function gj(a){switch(a.tag){case 5:return bc(a.type);case 16:return bc("Lazy");case 13:return bc("Suspense");case 19:return bc("SuspenseList");case 0:case 2:case 15:return a=be(a.type,!1),a;case 11:return a=be(a.type.render,!1),a;case 1:return a=be(a.type,!0),a;default:return""}}function de(a){if(null==a)return null;if("function"===typeof a)return a.displayName||a.name||null;if("string"===typeof a)return a;switch(a){case Bb:return"Fragment";case Cb:return"Portal";case ee:return"Profiler";case fe:return"StrictMode"; +case ge:return"Suspense";case he:return"SuspenseList"}if("object"===typeof a)switch(a.$$typeof){case gg:return(a.displayName||"Context")+".Consumer";case hg:return(a._context.displayName||"Context")+".Provider";case ie:var b=a.render;a=a.displayName;a||(a=b.displayName||b.name||"",a=""!==a?"ForwardRef("+a+")":"ForwardRef");return a;case je:return b=a.displayName||null,null!==b?b:de(a.type)||"Memo";case Ta:b=a._payload;a=a._init;try{return de(a(b))}catch(c){}}return null}function hj(a){var b=a.type; +switch(a.tag){case 24:return"Cache";case 9:return(b.displayName||"Context")+".Consumer";case 10:return(b._context.displayName||"Context")+".Provider";case 18:return"DehydratedFragment";case 11:return a=b.render,a=a.displayName||a.name||"",b.displayName||(""!==a?"ForwardRef("+a+")":"ForwardRef");case 7:return"Fragment";case 5:return b;case 4:return"Portal";case 3:return"Root";case 6:return"Text";case 16:return de(b);case 8:return b===fe?"StrictMode":"Mode";case 22:return"Offscreen";case 12:return"Profiler"; +case 21:return"Scope";case 13:return"Suspense";case 19:return"SuspenseList";case 25:return"TracingMarker";case 1:case 0:case 17:case 2:case 14:case 15:if("function"===typeof b)return b.displayName||b.name||null;if("string"===typeof b)return b}return null}function Ua(a){switch(typeof a){case "boolean":case "number":case "string":case "undefined":return a;case "object":return a;default:return""}}function ig(a){var b=a.type;return(a=a.nodeName)&&"input"===a.toLowerCase()&&("checkbox"===b||"radio"=== +b)}function ij(a){var b=ig(a)?"checked":"value",c=Object.getOwnPropertyDescriptor(a.constructor.prototype,b),d=""+a[b];if(!a.hasOwnProperty(b)&&"undefined"!==typeof c&&"function"===typeof c.get&&"function"===typeof c.set){var e=c.get,f=c.set;Object.defineProperty(a,b,{configurable:!0,get:function(){return e.call(this)},set:function(a){d=""+a;f.call(this,a)}});Object.defineProperty(a,b,{enumerable:c.enumerable});return{getValue:function(){return d},setValue:function(a){d=""+a},stopTracking:function(){a._valueTracker= +null;delete a[b]}}}}function Pc(a){a._valueTracker||(a._valueTracker=ij(a))}function jg(a){if(!a)return!1;var b=a._valueTracker;if(!b)return!0;var c=b.getValue();var d="";a&&(d=ig(a)?a.checked?"true":"false":a.value);a=d;return a!==c?(b.setValue(a),!0):!1}function Qc(a){a=a||("undefined"!==typeof document?document:void 0);if("undefined"===typeof a)return null;try{return a.activeElement||a.body}catch(b){return a.body}}function ke(a,b){var c=b.checked;return E({},b,{defaultChecked:void 0,defaultValue:void 0, +value:void 0,checked:null!=c?c:a._wrapperState.initialChecked})}function kg(a,b){var c=null==b.defaultValue?"":b.defaultValue,d=null!=b.checked?b.checked:b.defaultChecked;c=Ua(null!=b.value?b.value:c);a._wrapperState={initialChecked:d,initialValue:c,controlled:"checkbox"===b.type||"radio"===b.type?null!=b.checked:null!=b.value}}function lg(a,b){b=b.checked;null!=b&&$d(a,"checked",b,!1)}function le(a,b){lg(a,b);var c=Ua(b.value),d=b.type;if(null!=c)if("number"===d){if(0===c&&""===a.value||a.value!= +c)a.value=""+c}else a.value!==""+c&&(a.value=""+c);else if("submit"===d||"reset"===d){a.removeAttribute("value");return}b.hasOwnProperty("value")?me(a,b.type,c):b.hasOwnProperty("defaultValue")&&me(a,b.type,Ua(b.defaultValue));null==b.checked&&null!=b.defaultChecked&&(a.defaultChecked=!!b.defaultChecked)}function mg(a,b,c){if(b.hasOwnProperty("value")||b.hasOwnProperty("defaultValue")){var d=b.type;if(!("submit"!==d&&"reset"!==d||void 0!==b.value&&null!==b.value))return;b=""+a._wrapperState.initialValue; +c||b===a.value||(a.value=b);a.defaultValue=b}c=a.name;""!==c&&(a.name="");a.defaultChecked=!!a._wrapperState.initialChecked;""!==c&&(a.name=c)}function me(a,b,c){if("number"!==b||Qc(a.ownerDocument)!==a)null==c?a.defaultValue=""+a._wrapperState.initialValue:a.defaultValue!==""+c&&(a.defaultValue=""+c)}function Db(a,b,c,d){a=a.options;if(b){b={};for(var e=0;e>>=0;return 0===a?32:31-(rj(a)/sj|0)|0}function hc(a){switch(a&-a){case 1:return 1;case 2:return 2;case 4:return 4;case 8:return 8;case 16:return 16;case 32:return 32;case 64:case 128:case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:case 262144:case 524288:case 1048576:case 2097152:return a& +4194240;case 4194304:case 8388608:case 16777216:case 33554432:case 67108864:return a&130023424;case 134217728:return 134217728;case 268435456:return 268435456;case 536870912:return 536870912;case 1073741824:return 1073741824;default:return a}}function Vc(a,b){var c=a.pendingLanes;if(0===c)return 0;var d=0,e=a.suspendedLanes,f=a.pingedLanes,g=c&268435455;if(0!==g){var h=g&~e;0!==h?d=hc(h):(f&=g,0!==f&&(d=hc(f)))}else g=c&~e,0!==g?d=hc(g):0!==f&&(d=hc(f));if(0===d)return 0;if(0!==b&&b!==d&&0===(b&e)&& +(e=d&-d,f=b&-b,e>=f||16===e&&0!==(f&4194240)))return b;0!==(d&4)&&(d|=c&16);b=a.entangledLanes;if(0!==b)for(a=a.entanglements,b&=d;0c;c++)b.push(a); +return b}function ic(a,b,c){a.pendingLanes|=b;536870912!==b&&(a.suspendedLanes=0,a.pingedLanes=0);a=a.eventTimes;b=31-ta(b);a[b]=c}function vj(a,b){var c=a.pendingLanes&~b;a.pendingLanes=b;a.suspendedLanes=0;a.pingedLanes=0;a.expiredLanes&=b;a.mutableReadLanes&=b;a.entangledLanes&=b;b=a.entanglements;var d=a.eventTimes;for(a=a.expirationTimes;0=b)return{node:c,offset:b-a};a=d}a:{for(;c;){if(c.nextSibling){c=c.nextSibling;break a}c=c.parentNode}c=void 0}c=$g(c)}}function bh(a,b){return a&&b?a===b?!0:a&&3===a.nodeType?!1:b&&3===b.nodeType?bh(a,b.parentNode):"contains"in a?a.contains(b):a.compareDocumentPosition?!!(a.compareDocumentPosition(b)&16):!1:!1}function ch(){for(var a=window,b=Qc();b instanceof a.HTMLIFrameElement;){try{var c="string"===typeof b.contentWindow.location.href}catch(d){c=!1}if(c)a=b.contentWindow;else break; +b=Qc(a.document)}return b}function Ie(a){var b=a&&a.nodeName&&a.nodeName.toLowerCase();return b&&("input"===b&&("text"===a.type||"search"===a.type||"tel"===a.type||"url"===a.type||"password"===a.type)||"textarea"===b||"true"===a.contentEditable)}function Uj(a){var b=ch(),c=a.focusedElem,d=a.selectionRange;if(b!==c&&c&&c.ownerDocument&&bh(c.ownerDocument.documentElement,c)){if(null!==d&&Ie(c))if(b=d.start,a=d.end,void 0===a&&(a=b),"selectionStart"in c)c.selectionStart=b,c.selectionEnd=Math.min(a,c.value.length); +else if(a=(b=c.ownerDocument||document)&&b.defaultView||window,a.getSelection){a=a.getSelection();var e=c.textContent.length,f=Math.min(d.start,e);d=void 0===d.end?f:Math.min(d.end,e);!a.extend&&f>d&&(e=d,d=f,f=e);e=ah(c,f);var g=ah(c,d);e&&g&&(1!==a.rangeCount||a.anchorNode!==e.node||a.anchorOffset!==e.offset||a.focusNode!==g.node||a.focusOffset!==g.offset)&&(b=b.createRange(),b.setStart(e.node,e.offset),a.removeAllRanges(),f>d?(a.addRange(b),a.extend(g.node,g.offset)):(b.setEnd(g.node,g.offset), +a.addRange(b)))}b=[];for(a=c;a=a.parentNode;)1===a.nodeType&&b.push({element:a,left:a.scrollLeft,top:a.scrollTop});"function"===typeof c.focus&&c.focus();for(c=0;cMb||(a.current=Se[Mb],Se[Mb]=null,Mb--)} +function y(a,b,c){Mb++;Se[Mb]=a.current;a.current=b}function Nb(a,b){var c=a.type.contextTypes;if(!c)return cb;var d=a.stateNode;if(d&&d.__reactInternalMemoizedUnmaskedChildContext===b)return d.__reactInternalMemoizedMaskedChildContext;var e={},f;for(f in c)e[f]=b[f];d&&(a=a.stateNode,a.__reactInternalMemoizedUnmaskedChildContext=b,a.__reactInternalMemoizedMaskedChildContext=e);return e}function ea(a){a=a.childContextTypes;return null!==a&&void 0!==a}function th(a,b,c){if(J.current!==cb)throw Error(n(168)); +y(J,b);y(S,c)}function uh(a,b,c){var d=a.stateNode;b=b.childContextTypes;if("function"!==typeof d.getChildContext)return c;d=d.getChildContext();for(var e in d)if(!(e in b))throw Error(n(108,hj(a)||"Unknown",e));return E({},c,d)}function ld(a){a=(a=a.stateNode)&&a.__reactInternalMemoizedMergedChildContext||cb;qb=J.current;y(J,a);y(S,S.current);return!0}function vh(a,b,c){var d=a.stateNode;if(!d)throw Error(n(169));c?(a=uh(a,b,qb),d.__reactInternalMemoizedMergedChildContext=a,w(S),w(J),y(J,a)):w(S); +y(S,c)}function wh(a){null===La?La=[a]:La.push(a)}function kk(a){md=!0;wh(a)}function db(){if(!Te&&null!==La){Te=!0;var a=0,b=z;try{var c=La;for(z=1;a>=g;e-=g;Ma=1<<32-ta(b)+e|c<q?(v=l,l=null):v=l.sibling;var A=r(e,l,h[q],k);if(null===A){null===l&&(l=v);break}a&&l&&null===A.alternate&&b(e,l);g=f(A,g,q);null===m?n=A:m.sibling=A;m=A;l=v}if(q===h.length)return c(e,l),D&&rb(e,q),n;if(null===l){for(;qv?(A=q,q=null):A=q.sibling;var x=r(e,q,t.value,k);if(null===x){null===q&&(q=A);break}a&&q&&null===x.alternate&&b(e,q);g=f(x,g,v);null===l?m=x:l.sibling=x;l=x;q=A}if(t.done)return c(e,q),D&&rb(e,v),m; +if(null===q){for(;!t.done;v++,t=h.next())t=u(e,t.value,k),null!==t&&(g=f(t,g,v),null===l?m=t:l.sibling=t,l=t);D&&rb(e,v);return m}for(q=d(e,q);!t.done;v++,t=h.next())t=p(q,e,v,t.value,k),null!==t&&(a&&null!==t.alternate&&q.delete(null===t.key?v:t.key),g=f(t,g,v),null===l?m=t:l.sibling=t,l=t);a&&q.forEach(function(a){return b(e,a)});D&&rb(e,v);return m}function w(a,d,f,h){"object"===typeof f&&null!==f&&f.type===Bb&&null===f.key&&(f=f.props.children);if("object"===typeof f&&null!==f){switch(f.$$typeof){case xd:a:{for(var k= +f.key,m=d;null!==m;){if(m.key===k){k=f.type;if(k===Bb){if(7===m.tag){c(a,m.sibling);d=e(m,f.props.children);d.return=a;a=d;break a}}else if(m.elementType===k||"object"===typeof k&&null!==k&&k.$$typeof===Ta&&Kh(k)===m.type){c(a,m.sibling);d=e(m,f.props);d.ref=vc(a,m,f);d.return=a;a=d;break a}c(a,m);break}else b(a,m);m=m.sibling}f.type===Bb?(d=ub(f.props.children,a.mode,h,f.key),d.return=a,a=d):(h=wd(f.type,f.key,f.props,null,a.mode,h),h.ref=vc(a,d,f),h.return=a,a=h)}return g(a);case Cb:a:{for(m=f.key;null!== +d;){if(d.key===m)if(4===d.tag&&d.stateNode.containerInfo===f.containerInfo&&d.stateNode.implementation===f.implementation){c(a,d.sibling);d=e(d,f.children||[]);d.return=a;a=d;break a}else{c(a,d);break}else b(a,d);d=d.sibling}d=hf(f,a.mode,h);d.return=a;a=d}return g(a);case Ta:return m=f._init,w(a,d,m(f._payload),h)}if(cc(f))return x(a,d,f,h);if(ac(f))return F(a,d,f,h);vd(a,f)}return"string"===typeof f&&""!==f||"number"===typeof f?(f=""+f,null!==d&&6===d.tag?(c(a,d.sibling),d=e(d,f),d.return=a,a=d): +(c(a,d),d=gf(f,a.mode,h),d.return=a,a=d),g(a)):c(a,d)}return w}function vb(a){if(a===wc)throw Error(n(174));return a}function jf(a,b){y(xc,b);y(yc,a);y(Ea,wc);a=b.nodeType;switch(a){case 9:case 11:b=(b=b.documentElement)?b.namespaceURI:oe(null,"");break;default:a=8===a?b.parentNode:b,b=a.namespaceURI||null,a=a.tagName,b=oe(b,a)}w(Ea);y(Ea,b)}function Tb(a){w(Ea);w(yc);w(xc)}function Mh(a){vb(xc.current);var b=vb(Ea.current);var c=oe(b,a.type);b!==c&&(y(yc,a),y(Ea,c))}function kf(a){yc.current===a&& +(w(Ea),w(yc))}function yd(a){for(var b=a;null!==b;){if(13===b.tag){var c=b.memoizedState;if(null!==c&&(c=c.dehydrated,null===c||"$?"===c.data||"$!"===c.data))return b}else if(19===b.tag&&void 0!==b.memoizedProps.revealOrder){if(0!==(b.flags&128))return b}else if(null!==b.child){b.child.return=b;b=b.child;continue}if(b===a)break;for(;null===b.sibling;){if(null===b.return||b.return===a)return null;b=b.return}b.sibling.return=b.return;b=b.sibling}return null}function lf(){for(var a=0;ac?c:4;a(!0);var d=uf.transition;uf.transition={};try{a(!1),b()}finally{z=c,uf.transition=d}}function di(){return sa().memoizedState}function rk(a,b, +c){var d=hb(a);c={lane:d,action:c,hasEagerState:!1,eagerState:null,next:null};if(ei(a))fi(b,c);else if(c=Ch(a,b,c,d),null!==c){var e=Z();ya(c,a,d,e);gi(c,b,d)}}function pk(a,b,c){var d=hb(a),e={lane:d,action:c,hasEagerState:!1,eagerState:null,next:null};if(ei(a))fi(b,e);else{var f=a.alternate;if(0===a.lanes&&(null===f||0===f.lanes)&&(f=b.lastRenderedReducer,null!==f))try{var g=b.lastRenderedState,h=f(g,c);e.hasEagerState=!0;e.eagerState=h;if(ua(h,g)){var k=b.interleaved;null===k?(e.next=e,cf(b)): +(e.next=k.next,k.next=e);b.interleaved=e;return}}catch(m){}finally{}c=Ch(a,b,e,d);null!==c&&(e=Z(),ya(c,a,d,e),gi(c,b,d))}}function ei(a){var b=a.alternate;return a===C||null!==b&&b===C}function fi(a,b){zc=Bd=!0;var c=a.pending;null===c?b.next=b:(b.next=c.next,c.next=b);a.pending=b}function gi(a,b,c){if(0!==(c&4194240)){var d=b.lanes;d&=a.pendingLanes;c|=d;b.lanes=c;xe(a,c)}}function Ub(a,b){try{var c="",d=b;do c+=gj(d),d=d.return;while(d);var e=c}catch(f){e="\nError generating stack: "+f.message+ +"\n"+f.stack}return{value:a,source:b,stack:e,digest:null}}function vf(a,b,c){return{value:a,source:null,stack:null!=c?c:null,digest:null!=b?b:null}}function wf(a,b){try{console.error(b.value)}catch(c){setTimeout(function(){throw c;})}}function hi(a,b,c){c=Pa(-1,c);c.tag=3;c.payload={element:null};var d=b.value;c.callback=function(){Ed||(Ed=!0,xf=d);wf(a,b)};return c}function ii(a,b,c){c=Pa(-1,c);c.tag=3;var d=a.type.getDerivedStateFromError;if("function"===typeof d){var e=b.value;c.payload=function(){return d(e)}; +c.callback=function(){wf(a,b)}}var f=a.stateNode;null!==f&&"function"===typeof f.componentDidCatch&&(c.callback=function(){wf(a,b);"function"!==typeof d&&(null===ib?ib=new Set([this]):ib.add(this));var c=b.stack;this.componentDidCatch(b.value,{componentStack:null!==c?c:""})});return c}function ji(a,b,c){var d=a.pingCache;if(null===d){d=a.pingCache=new sk;var e=new Set;d.set(b,e)}else e=d.get(b),void 0===e&&(e=new Set,d.set(b,e));e.has(c)||(e.add(c),a=tk.bind(null,a,b,c),b.then(a,a))}function ki(a){do{var b; +if(b=13===a.tag)b=a.memoizedState,b=null!==b?null!==b.dehydrated?!0:!1:!0;if(b)return a;a=a.return}while(null!==a);return null}function li(a,b,c,d,e){if(0===(a.mode&1))return a===b?a.flags|=65536:(a.flags|=128,c.flags|=131072,c.flags&=-52805,1===c.tag&&(null===c.alternate?c.tag=17:(b=Pa(-1,1),b.tag=2,eb(c,b,1))),c.lanes|=1),a;a.flags|=65536;a.lanes=e;return a}function aa(a,b,c,d){b.child=null===a?mi(b,null,c,d):Vb(b,a.child,c,d)}function ni(a,b,c,d,e){c=c.render;var f=b.ref;Sb(b,e);d=of(a,b,c,d,f, +e);c=pf();if(null!==a&&!ha)return b.updateQueue=a.updateQueue,b.flags&=-2053,a.lanes&=~e,Qa(a,b,e);D&&c&&Ue(b);b.flags|=1;aa(a,b,d,e);return b.child}function oi(a,b,c,d,e){if(null===a){var f=c.type;if("function"===typeof f&&!yf(f)&&void 0===f.defaultProps&&null===c.compare&&void 0===c.defaultProps)return b.tag=15,b.type=f,pi(a,b,f,d,e);a=wd(c.type,null,d,b,b.mode,e);a.ref=b.ref;a.return=b;return b.child=a}f=a.child;if(0===(a.lanes&e)){var g=f.memoizedProps;c=c.compare;c=null!==c?c:qc;if(c(g,d)&&a.ref=== +b.ref)return Qa(a,b,e)}b.flags|=1;a=gb(f,d);a.ref=b.ref;a.return=b;return b.child=a}function pi(a,b,c,d,e){if(null!==a){var f=a.memoizedProps;if(qc(f,d)&&a.ref===b.ref)if(ha=!1,b.pendingProps=d=f,0!==(a.lanes&e))0!==(a.flags&131072)&&(ha=!0);else return b.lanes=a.lanes,Qa(a,b,e)}return zf(a,b,c,d,e)}function qi(a,b,c){var d=b.pendingProps,e=d.children,f=null!==a?a.memoizedState:null;if("hidden"===d.mode)if(0===(b.mode&1))b.memoizedState={baseLanes:0,cachePool:null,transitions:null},y(Ga,ba),ba|=c; +else{if(0===(c&1073741824))return a=null!==f?f.baseLanes|c:c,b.lanes=b.childLanes=1073741824,b.memoizedState={baseLanes:a,cachePool:null,transitions:null},b.updateQueue=null,y(Ga,ba),ba|=a,null;b.memoizedState={baseLanes:0,cachePool:null,transitions:null};d=null!==f?f.baseLanes:c;y(Ga,ba);ba|=d}else null!==f?(d=f.baseLanes|c,b.memoizedState=null):d=c,y(Ga,ba),ba|=d;aa(a,b,e,c);return b.child}function ri(a,b){var c=b.ref;if(null===a&&null!==c||null!==a&&a.ref!==c)b.flags|=512,b.flags|=2097152}function zf(a, +b,c,d,e){var f=ea(c)?qb:J.current;f=Nb(b,f);Sb(b,e);c=of(a,b,c,d,f,e);d=pf();if(null!==a&&!ha)return b.updateQueue=a.updateQueue,b.flags&=-2053,a.lanes&=~e,Qa(a,b,e);D&&d&&Ue(b);b.flags|=1;aa(a,b,c,e);return b.child}function si(a,b,c,d,e){if(ea(c)){var f=!0;ld(b)}else f=!1;Sb(b,e);if(null===b.stateNode)Fd(a,b),Hh(b,c,d),ff(b,c,d,e),d=!0;else if(null===a){var g=b.stateNode,h=b.memoizedProps;g.props=h;var k=g.context,m=c.contextType;"object"===typeof m&&null!==m?m=qa(m):(m=ea(c)?qb:J.current,m=Nb(b, +m));var l=c.getDerivedStateFromProps,n="function"===typeof l||"function"===typeof g.getSnapshotBeforeUpdate;n||"function"!==typeof g.UNSAFE_componentWillReceiveProps&&"function"!==typeof g.componentWillReceiveProps||(h!==d||k!==m)&&Ih(b,g,d,m);fb=!1;var r=b.memoizedState;g.state=r;td(b,d,g,e);k=b.memoizedState;h!==d||r!==k||S.current||fb?("function"===typeof l&&(ef(b,c,l,d),k=b.memoizedState),(h=fb||Gh(b,c,h,d,r,k,m))?(n||"function"!==typeof g.UNSAFE_componentWillMount&&"function"!==typeof g.componentWillMount|| +("function"===typeof g.componentWillMount&&g.componentWillMount(),"function"===typeof g.UNSAFE_componentWillMount&&g.UNSAFE_componentWillMount()),"function"===typeof g.componentDidMount&&(b.flags|=4194308)):("function"===typeof g.componentDidMount&&(b.flags|=4194308),b.memoizedProps=d,b.memoizedState=k),g.props=d,g.state=k,g.context=m,d=h):("function"===typeof g.componentDidMount&&(b.flags|=4194308),d=!1)}else{g=b.stateNode;Dh(a,b);h=b.memoizedProps;m=b.type===b.elementType?h:xa(b.type,h);g.props= +m;n=b.pendingProps;r=g.context;k=c.contextType;"object"===typeof k&&null!==k?k=qa(k):(k=ea(c)?qb:J.current,k=Nb(b,k));var p=c.getDerivedStateFromProps;(l="function"===typeof p||"function"===typeof g.getSnapshotBeforeUpdate)||"function"!==typeof g.UNSAFE_componentWillReceiveProps&&"function"!==typeof g.componentWillReceiveProps||(h!==n||r!==k)&&Ih(b,g,d,k);fb=!1;r=b.memoizedState;g.state=r;td(b,d,g,e);var x=b.memoizedState;h!==n||r!==x||S.current||fb?("function"===typeof p&&(ef(b,c,p,d),x=b.memoizedState), +(m=fb||Gh(b,c,m,d,r,x,k)||!1)?(l||"function"!==typeof g.UNSAFE_componentWillUpdate&&"function"!==typeof g.componentWillUpdate||("function"===typeof g.componentWillUpdate&&g.componentWillUpdate(d,x,k),"function"===typeof g.UNSAFE_componentWillUpdate&&g.UNSAFE_componentWillUpdate(d,x,k)),"function"===typeof g.componentDidUpdate&&(b.flags|=4),"function"===typeof g.getSnapshotBeforeUpdate&&(b.flags|=1024)):("function"!==typeof g.componentDidUpdate||h===a.memoizedProps&&r===a.memoizedState||(b.flags|= +4),"function"!==typeof g.getSnapshotBeforeUpdate||h===a.memoizedProps&&r===a.memoizedState||(b.flags|=1024),b.memoizedProps=d,b.memoizedState=x),g.props=d,g.state=x,g.context=k,d=m):("function"!==typeof g.componentDidUpdate||h===a.memoizedProps&&r===a.memoizedState||(b.flags|=4),"function"!==typeof g.getSnapshotBeforeUpdate||h===a.memoizedProps&&r===a.memoizedState||(b.flags|=1024),d=!1)}return Af(a,b,c,d,f,e)}function Af(a,b,c,d,e,f){ri(a,b);var g=0!==(b.flags&128);if(!d&&!g)return e&&vh(b,c,!1), +Qa(a,b,f);d=b.stateNode;uk.current=b;var h=g&&"function"!==typeof c.getDerivedStateFromError?null:d.render();b.flags|=1;null!==a&&g?(b.child=Vb(b,a.child,null,f),b.child=Vb(b,null,h,f)):aa(a,b,h,f);b.memoizedState=d.state;e&&vh(b,c,!0);return b.child}function ti(a){var b=a.stateNode;b.pendingContext?th(a,b.pendingContext,b.pendingContext!==b.context):b.context&&th(a,b.context,!1);jf(a,b.containerInfo)}function ui(a,b,c,d,e){Qb();Ye(e);b.flags|=256;aa(a,b,c,d);return b.child}function Bf(a){return{baseLanes:a, +cachePool:null,transitions:null}}function vi(a,b,c){var d=b.pendingProps,e=G.current,f=!1,g=0!==(b.flags&128),h;(h=g)||(h=null!==a&&null===a.memoizedState?!1:0!==(e&2));if(h)f=!0,b.flags&=-129;else if(null===a||null!==a.memoizedState)e|=1;y(G,e&1);if(null===a){Xe(b);a=b.memoizedState;if(null!==a&&(a=a.dehydrated,null!==a))return 0===(b.mode&1)?b.lanes=1:"$!"===a.data?b.lanes=8:b.lanes=1073741824,null;g=d.children;a=d.fallback;return f?(d=b.mode,f=b.child,g={mode:"hidden",children:g},0===(d&1)&&null!== +f?(f.childLanes=0,f.pendingProps=g):f=Gd(g,d,0,null),a=ub(a,d,c,null),f.return=b,a.return=b,f.sibling=a,b.child=f,b.child.memoizedState=Bf(c),b.memoizedState=Cf,a):Df(b,g)}e=a.memoizedState;if(null!==e&&(h=e.dehydrated,null!==h))return vk(a,b,g,d,h,e,c);if(f){f=d.fallback;g=b.mode;e=a.child;h=e.sibling;var k={mode:"hidden",children:d.children};0===(g&1)&&b.child!==e?(d=b.child,d.childLanes=0,d.pendingProps=k,b.deletions=null):(d=gb(e,k),d.subtreeFlags=e.subtreeFlags&14680064);null!==h?f=gb(h,f):(f= +ub(f,g,c,null),f.flags|=2);f.return=b;d.return=b;d.sibling=f;b.child=d;d=f;f=b.child;g=a.child.memoizedState;g=null===g?Bf(c):{baseLanes:g.baseLanes|c,cachePool:null,transitions:g.transitions};f.memoizedState=g;f.childLanes=a.childLanes&~c;b.memoizedState=Cf;return d}f=a.child;a=f.sibling;d=gb(f,{mode:"visible",children:d.children});0===(b.mode&1)&&(d.lanes=c);d.return=b;d.sibling=null;null!==a&&(c=b.deletions,null===c?(b.deletions=[a],b.flags|=16):c.push(a));b.child=d;b.memoizedState=null;return d} +function Df(a,b,c){b=Gd({mode:"visible",children:b},a.mode,0,null);b.return=a;return a.child=b}function Hd(a,b,c,d){null!==d&&Ye(d);Vb(b,a.child,null,c);a=Df(b,b.pendingProps.children);a.flags|=2;b.memoizedState=null;return a}function vk(a,b,c,d,e,f,g){if(c){if(b.flags&256)return b.flags&=-257,d=vf(Error(n(422))),Hd(a,b,g,d);if(null!==b.memoizedState)return b.child=a.child,b.flags|=128,null;f=d.fallback;e=b.mode;d=Gd({mode:"visible",children:d.children},e,0,null);f=ub(f,e,g,null);f.flags|=2;d.return= +b;f.return=b;d.sibling=f;b.child=d;0!==(b.mode&1)&&Vb(b,a.child,null,g);b.child.memoizedState=Bf(g);b.memoizedState=Cf;return f}if(0===(b.mode&1))return Hd(a,b,g,null);if("$!"===e.data){d=e.nextSibling&&e.nextSibling.dataset;if(d)var h=d.dgst;d=h;f=Error(n(419));d=vf(f,d,void 0);return Hd(a,b,g,d)}h=0!==(g&a.childLanes);if(ha||h){d=O;if(null!==d){switch(g&-g){case 4:e=2;break;case 16:e=8;break;case 64:case 128:case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:case 262144:case 524288:case 1048576:case 2097152:case 4194304:case 8388608:case 16777216:case 33554432:case 67108864:e= +32;break;case 536870912:e=268435456;break;default:e=0}e=0!==(e&(d.suspendedLanes|g))?0:e;0!==e&&e!==f.retryLane&&(f.retryLane=e,Oa(a,e),ya(d,a,e,-1))}Ef();d=vf(Error(n(421)));return Hd(a,b,g,d)}if("$?"===e.data)return b.flags|=128,b.child=a.child,b=wk.bind(null,a),e._reactRetry=b,null;a=f.treeContext;fa=Ka(e.nextSibling);la=b;D=!0;wa=null;null!==a&&(na[oa++]=Ma,na[oa++]=Na,na[oa++]=sb,Ma=a.id,Na=a.overflow,sb=b);b=Df(b,d.children);b.flags|=4096;return b}function wi(a,b,c){a.lanes|=b;var d=a.alternate; +null!==d&&(d.lanes|=b);bf(a.return,b,c)}function Ff(a,b,c,d,e){var f=a.memoizedState;null===f?a.memoizedState={isBackwards:b,rendering:null,renderingStartTime:0,last:d,tail:c,tailMode:e}:(f.isBackwards=b,f.rendering=null,f.renderingStartTime=0,f.last=d,f.tail=c,f.tailMode=e)}function xi(a,b,c){var d=b.pendingProps,e=d.revealOrder,f=d.tail;aa(a,b,d.children,c);d=G.current;if(0!==(d&2))d=d&1|2,b.flags|=128;else{if(null!==a&&0!==(a.flags&128))a:for(a=b.child;null!==a;){if(13===a.tag)null!==a.memoizedState&& +wi(a,c,b);else if(19===a.tag)wi(a,c,b);else if(null!==a.child){a.child.return=a;a=a.child;continue}if(a===b)break a;for(;null===a.sibling;){if(null===a.return||a.return===b)break a;a=a.return}a.sibling.return=a.return;a=a.sibling}d&=1}y(G,d);if(0===(b.mode&1))b.memoizedState=null;else switch(e){case "forwards":c=b.child;for(e=null;null!==c;)a=c.alternate,null!==a&&null===yd(a)&&(e=c),c=c.sibling;c=e;null===c?(e=b.child,b.child=null):(e=c.sibling,c.sibling=null);Ff(b,!1,e,c,f);break;case "backwards":c= +null;e=b.child;for(b.child=null;null!==e;){a=e.alternate;if(null!==a&&null===yd(a)){b.child=e;break}a=e.sibling;e.sibling=c;c=e;e=a}Ff(b,!0,c,null,f);break;case "together":Ff(b,!1,null,null,void 0);break;default:b.memoizedState=null}return b.child}function Fd(a,b){0===(b.mode&1)&&null!==a&&(a.alternate=null,b.alternate=null,b.flags|=2)}function Qa(a,b,c){null!==a&&(b.dependencies=a.dependencies);ra|=b.lanes;if(0===(c&b.childLanes))return null;if(null!==a&&b.child!==a.child)throw Error(n(153));if(null!== +b.child){a=b.child;c=gb(a,a.pendingProps);b.child=c;for(c.return=b;null!==a.sibling;)a=a.sibling,c=c.sibling=gb(a,a.pendingProps),c.return=b;c.sibling=null}return b.child}function xk(a,b,c){switch(b.tag){case 3:ti(b);Qb();break;case 5:Mh(b);break;case 1:ea(b.type)&&ld(b);break;case 4:jf(b,b.stateNode.containerInfo);break;case 10:var d=b.type._context,e=b.memoizedProps.value;y(rd,d._currentValue);d._currentValue=e;break;case 13:d=b.memoizedState;if(null!==d){if(null!==d.dehydrated)return y(G,G.current& +1),b.flags|=128,null;if(0!==(c&b.child.childLanes))return vi(a,b,c);y(G,G.current&1);a=Qa(a,b,c);return null!==a?a.sibling:null}y(G,G.current&1);break;case 19:d=0!==(c&b.childLanes);if(0!==(a.flags&128)){if(d)return xi(a,b,c);b.flags|=128}e=b.memoizedState;null!==e&&(e.rendering=null,e.tail=null,e.lastEffect=null);y(G,G.current);if(d)break;else return null;case 22:case 23:return b.lanes=0,qi(a,b,c)}return Qa(a,b,c)}function Dc(a,b){if(!D)switch(a.tailMode){case "hidden":b=a.tail;for(var c=null;null!== +b;)null!==b.alternate&&(c=b),b=b.sibling;null===c?a.tail=null:c.sibling=null;break;case "collapsed":c=a.tail;for(var d=null;null!==c;)null!==c.alternate&&(d=c),c=c.sibling;null===d?b||null===a.tail?a.tail=null:a.tail.sibling=null:d.sibling=null}}function W(a){var b=null!==a.alternate&&a.alternate.child===a.child,c=0,d=0;if(b)for(var e=a.child;null!==e;)c|=e.lanes|e.childLanes,d|=e.subtreeFlags&14680064,d|=e.flags&14680064,e.return=a,e=e.sibling;else for(e=a.child;null!==e;)c|=e.lanes|e.childLanes, +d|=e.subtreeFlags,d|=e.flags,e.return=a,e=e.sibling;a.subtreeFlags|=d;a.childLanes=c;return b}function yk(a,b,c){var d=b.pendingProps;Ve(b);switch(b.tag){case 2:case 16:case 15:case 0:case 11:case 7:case 8:case 12:case 9:case 14:return W(b),null;case 1:return ea(b.type)&&(w(S),w(J)),W(b),null;case 3:d=b.stateNode;Tb();w(S);w(J);lf();d.pendingContext&&(d.context=d.pendingContext,d.pendingContext=null);if(null===a||null===a.child)pd(b)?b.flags|=4:null===a||a.memoizedState.isDehydrated&&0===(b.flags& +256)||(b.flags|=1024,null!==wa&&(Gf(wa),wa=null));yi(a,b);W(b);return null;case 5:kf(b);var e=vb(xc.current);c=b.type;if(null!==a&&null!=b.stateNode)zk(a,b,c,d,e),a.ref!==b.ref&&(b.flags|=512,b.flags|=2097152);else{if(!d){if(null===b.stateNode)throw Error(n(166));W(b);return null}a=vb(Ea.current);if(pd(b)){d=b.stateNode;c=b.type;var f=b.memoizedProps;d[Da]=b;d[uc]=f;a=0!==(b.mode&1);switch(c){case "dialog":B("cancel",d);B("close",d);break;case "iframe":case "object":case "embed":B("load",d);break; +case "video":case "audio":for(e=0;e\x3c/script>",a=a.removeChild(a.firstChild)):"string"===typeof d.is?a=g.createElement(c,{is:d.is}):(a=g.createElement(c),"select"===c&&(g=a,d.multiple?g.multiple=!0:d.size&&(g.size=d.size))):a=g.createElementNS(a,c);a[Da]=b;a[uc]=d;Ak(a,b,!1,!1);b.stateNode=a;a:{g=qe(c,d);switch(c){case "dialog":B("cancel",a);B("close",a);e=d;break;case "iframe":case "object":case "embed":B("load",a);e=d;break; +case "video":case "audio":for(e=0;eHf&&(b.flags|=128,d=!0,Dc(f,!1),b.lanes=4194304)}else{if(!d)if(a=yd(g),null!==a){if(b.flags|=128,d=!0,c=a.updateQueue,null!==c&&(b.updateQueue=c,b.flags|=4),Dc(f,!0),null===f.tail&&"hidden"===f.tailMode&&!g.alternate&&!D)return W(b),null}else 2*P()-f.renderingStartTime>Hf&&1073741824!==c&&(b.flags|= +128,d=!0,Dc(f,!1),b.lanes=4194304);f.isBackwards?(g.sibling=b.child,b.child=g):(c=f.last,null!==c?c.sibling=g:b.child=g,f.last=g)}if(null!==f.tail)return b=f.tail,f.rendering=b,f.tail=b.sibling,f.renderingStartTime=P(),b.sibling=null,c=G.current,y(G,d?c&1|2:c&1),b;W(b);return null;case 22:case 23:return ba=Ga.current,w(Ga),d=null!==b.memoizedState,null!==a&&null!==a.memoizedState!==d&&(b.flags|=8192),d&&0!==(b.mode&1)?0!==(ba&1073741824)&&(W(b),b.subtreeFlags&6&&(b.flags|=8192)):W(b),null;case 24:return null; +case 25:return null}throw Error(n(156,b.tag));}function Ck(a,b,c){Ve(b);switch(b.tag){case 1:return ea(b.type)&&(w(S),w(J)),a=b.flags,a&65536?(b.flags=a&-65537|128,b):null;case 3:return Tb(),w(S),w(J),lf(),a=b.flags,0!==(a&65536)&&0===(a&128)?(b.flags=a&-65537|128,b):null;case 5:return kf(b),null;case 13:w(G);a=b.memoizedState;if(null!==a&&null!==a.dehydrated){if(null===b.alternate)throw Error(n(340));Qb()}a=b.flags;return a&65536?(b.flags=a&-65537|128,b):null;case 19:return w(G),null;case 4:return Tb(), +null;case 10:return af(b.type._context),null;case 22:case 23:return ba=Ga.current,w(Ga),null;case 24:return null;default:return null}}function Wb(a,b){var c=a.ref;if(null!==c)if("function"===typeof c)try{c(null)}catch(d){H(a,b,d)}else c.current=null}function If(a,b,c){try{c()}catch(d){H(a,b,d)}}function Dk(a,b){Jf=Zc;a=ch();if(Ie(a)){if("selectionStart"in a)var c={start:a.selectionStart,end:a.selectionEnd};else a:{c=(c=a.ownerDocument)&&c.defaultView||window;var d=c.getSelection&&c.getSelection(); +if(d&&0!==d.rangeCount){c=d.anchorNode;var e=d.anchorOffset,f=d.focusNode;d=d.focusOffset;try{c.nodeType,f.nodeType}catch(M){c=null;break a}var g=0,h=-1,k=-1,m=0,t=0,u=a,r=null;b:for(;;){for(var p;;){u!==c||0!==e&&3!==u.nodeType||(h=g+e);u!==f||0!==d&&3!==u.nodeType||(k=g+d);3===u.nodeType&&(g+=u.nodeValue.length);if(null===(p=u.firstChild))break;r=u;u=p}for(;;){if(u===a)break b;r===c&&++m===e&&(h=g);r===f&&++t===d&&(k=g);if(null!==(p=u.nextSibling))break;u=r;r=u.parentNode}u=p}c=-1===h||-1===k?null: +{start:h,end:k}}else c=null}c=c||{start:0,end:0}}else c=null;Kf={focusedElem:a,selectionRange:c};Zc=!1;for(l=b;null!==l;)if(b=l,a=b.child,0!==(b.subtreeFlags&1028)&&null!==a)a.return=b,l=a;else for(;null!==l;){b=l;try{var x=b.alternate;if(0!==(b.flags&1024))switch(b.tag){case 0:case 11:case 15:break;case 1:if(null!==x){var w=x.memoizedProps,z=x.memoizedState,A=b.stateNode,v=A.getSnapshotBeforeUpdate(b.elementType===b.type?w:xa(b.type,w),z);A.__reactInternalSnapshotBeforeUpdate=v}break;case 3:var q= +b.stateNode.containerInfo;1===q.nodeType?q.textContent="":9===q.nodeType&&q.documentElement&&q.removeChild(q.documentElement);break;case 5:case 6:case 4:case 17:break;default:throw Error(n(163));}}catch(M){H(b,b.return,M)}a=b.sibling;if(null!==a){a.return=b.return;l=a;break}l=b.return}x=Ai;Ai=!1;return x}function Gc(a,b,c){var d=b.updateQueue;d=null!==d?d.lastEffect:null;if(null!==d){var e=d=d.next;do{if((e.tag&a)===a){var f=e.destroy;e.destroy=void 0;void 0!==f&&If(b,c,f)}e=e.next}while(e!==d)}} +function Id(a,b){b=b.updateQueue;b=null!==b?b.lastEffect:null;if(null!==b){var c=b=b.next;do{if((c.tag&a)===a){var d=c.create;c.destroy=d()}c=c.next}while(c!==b)}}function Lf(a){var b=a.ref;if(null!==b){var c=a.stateNode;switch(a.tag){case 5:a=c;break;default:a=c}"function"===typeof b?b(a):b.current=a}}function Bi(a){var b=a.alternate;null!==b&&(a.alternate=null,Bi(b));a.child=null;a.deletions=null;a.sibling=null;5===a.tag&&(b=a.stateNode,null!==b&&(delete b[Da],delete b[uc],delete b[Me],delete b[Ek], +delete b[Fk]));a.stateNode=null;a.return=null;a.dependencies=null;a.memoizedProps=null;a.memoizedState=null;a.pendingProps=null;a.stateNode=null;a.updateQueue=null}function Ci(a){return 5===a.tag||3===a.tag||4===a.tag}function Di(a){a:for(;;){for(;null===a.sibling;){if(null===a.return||Ci(a.return))return null;a=a.return}a.sibling.return=a.return;for(a=a.sibling;5!==a.tag&&6!==a.tag&&18!==a.tag;){if(a.flags&2)continue a;if(null===a.child||4===a.tag)continue a;else a.child.return=a,a=a.child}if(!(a.flags& +2))return a.stateNode}}function Mf(a,b,c){var d=a.tag;if(5===d||6===d)a=a.stateNode,b?8===c.nodeType?c.parentNode.insertBefore(a,b):c.insertBefore(a,b):(8===c.nodeType?(b=c.parentNode,b.insertBefore(a,c)):(b=c,b.appendChild(a)),c=c._reactRootContainer,null!==c&&void 0!==c||null!==b.onclick||(b.onclick=kd));else if(4!==d&&(a=a.child,null!==a))for(Mf(a,b,c),a=a.sibling;null!==a;)Mf(a,b,c),a=a.sibling}function Nf(a,b,c){var d=a.tag;if(5===d||6===d)a=a.stateNode,b?c.insertBefore(a,b):c.appendChild(a); +else if(4!==d&&(a=a.child,null!==a))for(Nf(a,b,c),a=a.sibling;null!==a;)Nf(a,b,c),a=a.sibling}function jb(a,b,c){for(c=c.child;null!==c;)Ei(a,b,c),c=c.sibling}function Ei(a,b,c){if(Ca&&"function"===typeof Ca.onCommitFiberUnmount)try{Ca.onCommitFiberUnmount(Uc,c)}catch(h){}switch(c.tag){case 5:X||Wb(c,b);case 6:var d=T,e=za;T=null;jb(a,b,c);T=d;za=e;null!==T&&(za?(a=T,c=c.stateNode,8===a.nodeType?a.parentNode.removeChild(c):a.removeChild(c)):T.removeChild(c.stateNode));break;case 18:null!==T&&(za? +(a=T,c=c.stateNode,8===a.nodeType?Re(a.parentNode,c):1===a.nodeType&&Re(a,c),nc(a)):Re(T,c.stateNode));break;case 4:d=T;e=za;T=c.stateNode.containerInfo;za=!0;jb(a,b,c);T=d;za=e;break;case 0:case 11:case 14:case 15:if(!X&&(d=c.updateQueue,null!==d&&(d=d.lastEffect,null!==d))){e=d=d.next;do{var f=e,g=f.destroy;f=f.tag;void 0!==g&&(0!==(f&2)?If(c,b,g):0!==(f&4)&&If(c,b,g));e=e.next}while(e!==d)}jb(a,b,c);break;case 1:if(!X&&(Wb(c,b),d=c.stateNode,"function"===typeof d.componentWillUnmount))try{d.props= +c.memoizedProps,d.state=c.memoizedState,d.componentWillUnmount()}catch(h){H(c,b,h)}jb(a,b,c);break;case 21:jb(a,b,c);break;case 22:c.mode&1?(X=(d=X)||null!==c.memoizedState,jb(a,b,c),X=d):jb(a,b,c);break;default:jb(a,b,c)}}function Fi(a){var b=a.updateQueue;if(null!==b){a.updateQueue=null;var c=a.stateNode;null===c&&(c=a.stateNode=new Gk);b.forEach(function(b){var d=Hk.bind(null,a,b);c.has(b)||(c.add(b),b.then(d,d))})}}function Aa(a,b,c){c=b.deletions;if(null!==c)for(var d=0;de&&(e=g);d&=~f}d=e;d=P()-d;d=(120>d?120:480>d?480:1080>d?1080:1920>d?1920:3E3>d?3E3:4320>d?4320:1960*Nk(d/1960))-d;if(10a?16:a;if(null===lb)var d=!1;else{a=lb;lb=null;Qd=0;if(0!==(p&6))throw Error(n(331));var e=p;p|=4;for(l=a.current;null!==l;){var f=l,g=f.child;if(0!==(l.flags&16)){var h=f.deletions;if(null!==h){for(var k=0;kP()-Of?xb(a,0):Sf|=c);ia(a,b)}function Ui(a,b){0===b&&(0===(a.mode&1)?b=1:(b=Rd,Rd<<=1,0===(Rd&130023424)&&(Rd=4194304)));var c=Z();a=Oa(a,b);null!==a&&(ic(a,b,c),ia(a,c))}function wk(a){var b=a.memoizedState,c=0;null!==b&&(c=b.retryLane);Ui(a,c)}function Hk(a,b){var c=0;switch(a.tag){case 13:var d=a.stateNode;var e=a.memoizedState;null!==e&&(c=e.retryLane); +break;case 19:d=a.stateNode;break;default:throw Error(n(314));}null!==d&&d.delete(b);Ui(a,c)}function Ni(a,b){return xh(a,b)}function Uk(a,b,c,d){this.tag=a;this.key=c;this.sibling=this.child=this.return=this.stateNode=this.type=this.elementType=null;this.index=0;this.ref=null;this.pendingProps=b;this.dependencies=this.memoizedState=this.updateQueue=this.memoizedProps=null;this.mode=d;this.subtreeFlags=this.flags=0;this.deletions=null;this.childLanes=this.lanes=0;this.alternate=null}function yf(a){a= +a.prototype;return!(!a||!a.isReactComponent)}function Vk(a){if("function"===typeof a)return yf(a)?1:0;if(void 0!==a&&null!==a){a=a.$$typeof;if(a===ie)return 11;if(a===je)return 14}return 2}function gb(a,b){var c=a.alternate;null===c?(c=pa(a.tag,b,a.key,a.mode),c.elementType=a.elementType,c.type=a.type,c.stateNode=a.stateNode,c.alternate=a,a.alternate=c):(c.pendingProps=b,c.type=a.type,c.flags=0,c.subtreeFlags=0,c.deletions=null);c.flags=a.flags&14680064;c.childLanes=a.childLanes;c.lanes=a.lanes;c.child= +a.child;c.memoizedProps=a.memoizedProps;c.memoizedState=a.memoizedState;c.updateQueue=a.updateQueue;b=a.dependencies;c.dependencies=null===b?null:{lanes:b.lanes,firstContext:b.firstContext};c.sibling=a.sibling;c.index=a.index;c.ref=a.ref;return c}function wd(a,b,c,d,e,f){var g=2;d=a;if("function"===typeof a)yf(a)&&(g=1);else if("string"===typeof a)g=5;else a:switch(a){case Bb:return ub(c.children,e,f,b);case fe:g=8;e|=8;break;case ee:return a=pa(12,c,b,e|2),a.elementType=ee,a.lanes=f,a;case ge:return a= +pa(13,c,b,e),a.elementType=ge,a.lanes=f,a;case he:return a=pa(19,c,b,e),a.elementType=he,a.lanes=f,a;case Vi:return Gd(c,e,f,b);default:if("object"===typeof a&&null!==a)switch(a.$$typeof){case hg:g=10;break a;case gg:g=9;break a;case ie:g=11;break a;case je:g=14;break a;case Ta:g=16;d=null;break a}throw Error(n(130,null==a?a:typeof a,""));}b=pa(g,c,b,e);b.elementType=a;b.type=d;b.lanes=f;return b}function ub(a,b,c,d){a=pa(7,a,d,b);a.lanes=c;return a}function Gd(a,b,c,d){a=pa(22,a,d,b);a.elementType= +Vi;a.lanes=c;a.stateNode={isHidden:!1};return a}function gf(a,b,c){a=pa(6,a,null,b);a.lanes=c;return a}function hf(a,b,c){b=pa(4,null!==a.children?a.children:[],a.key,b);b.lanes=c;b.stateNode={containerInfo:a.containerInfo,pendingChildren:null,implementation:a.implementation};return b}function Wk(a,b,c,d,e){this.tag=b;this.containerInfo=a;this.finishedWork=this.pingCache=this.current=this.pendingChildren=null;this.timeoutHandle=-1;this.callbackNode=this.pendingContext=this.context=null;this.callbackPriority= +0;this.eventTimes=we(0);this.expirationTimes=we(-1);this.entangledLanes=this.finishedLanes=this.mutableReadLanes=this.expiredLanes=this.pingedLanes=this.suspendedLanes=this.pendingLanes=0;this.entanglements=we(0);this.identifierPrefix=d;this.onRecoverableError=e;this.mutableSourceEagerHydrationData=null}function Vf(a,b,c,d,e,f,g,h,k,m){a=new Wk(a,b,c,h,k);1===b?(b=1,!0===f&&(b|=8)):b=0;f=pa(3,null,null,b);a.current=f;f.stateNode=a;f.memoizedState={element:d,isDehydrated:c,cache:null,transitions:null, +pendingSuspenseBoundaries:null};df(f);return a}function Xk(a,b,c){var d=3