diff --git a/LICENSES.md b/LICENSES.md index 329a2573c8dbd9b08b0c02764ecc0de3b00ce03d..b7b901807275ee2e681ef3a207ca0b523f0a2a92 100644 --- a/LICENSES.md +++ b/LICENSES.md @@ -33,6 +33,7 @@ third-party components, and every **pretrained checkpoint / backbone** SynthScop | `cure` | CuRe (arXiv 2609.14316) | manic-cui/CuRe | Apache-2.0 (repo `LICENSE`) | **official** 25 MB adapter (`cure_adapter.pt`, loads as `cure.pth`) | Apache-2.0 (shipped in the repo; verify before redistributing) | | `d3qe` | D3QE (ICCV'25) | Zhangyr2022/D3QE | MIT (repo `LICENSE`); `official/` vendors its network code — see the file headers | **official** checkpoint (`model_epoch_best.pth` from HF `Yanran21/D3QE`, loads as `d3qe.pth`) | MIT (HF model card license tag) | | `dda` | DDA (NeurIPS'25) | roy-ch/Dual-Data-Alignment | Apache-2.0 (repo `LICENSE`); `official_arch.py` vendors its `lora.py` — see its header | **official** checkpoint (`DDA_ckpt.pth` from HF `Junwei-Xi/Dual-Data-Alignment`, loads as `dda.pth`) | Apache-2.0 (HF model card license tag) | +| `omniaid`, `omniaid_v1`, `omniaid_genimage` | OmniAID (arXiv 2511.08423) | yunncheng/OmniAID | MIT (README badge; no LICENSE file in the repo); `official_arch.py` vendors `models/OmniAID.py` — see its header | **official** checkpoints (`ckpt/checkpoint_omniaid_{v2,v1,genimage_sd14}.pth` from HF `Yunncheng/OmniAID`, loaded as `.pth`) | MIT (HF model card license tag) | | `dinov2` | DINOv2 probe | facebookresearch/dinov2 | Apache-2.0 (verify) | DINOv2 backbone | Apache-2.0 (verify) | | `dinov3` | DINOv3 probe | facebookresearch/dinov3 | DINOv3 License (gated) | DINOv3 backbone | **gated** — DINOv3 License | | `adapted_dino` | Adapter-tuned DINOv2 (SynthScope) | this repo | Apache-2.0 | — (SynthScope-trained) | n/a | @@ -45,6 +46,7 @@ third-party components, and every **pretrained checkpoint / backbone** SynthScop | Backbone | Used by | License | | --- | --- | --- | | OpenAI CLIP (ViT-L/14) | `clip`, `fat_former`, `d3`, `d3qe` | MIT (verify) | +| OpenAI CLIP ViT-L/14@336 (HF `openai/clip-vit-large-patch14-336`) | `omniaid`, `omniaid_v1`, `omniaid_genimage` | MIT | | OpenCLIP (ConvNeXt-XXL, LAION) | `aide` | code MIT; weights per LAION / model card (verify) | | DINOv2 (ViT-L/14, ViT-S/14) | `dinov2`, `adapted_dino`, `restrav`, `dda` | Apache-2.0 (verify) | | DINOv3 (ViT-L/16) | `dinov3`, `stall` | **gated** — DINOv3 License (non-commercial; accept upstream) | diff --git a/conf/dataset/image/arforensics.yaml b/conf/dataset/image/arforensics.yaml index dd2ad7b7580b61a104d7e21f4a1a2594912d748b..995bb383dea580904ae4456f7f6c46577bbf6468 100644 --- a/conf/dataset/image/arforensics.yaml +++ b/conf/dataset/image/arforensics.yaml @@ -3,6 +3,8 @@ name: arforensics # Open-MAGVIT2), 6000 fakes + 6000 ImageNet reals each. (HF Yanran21/ARForensics, corrected test release, 21 GB, MIT) root_dir: /hddstore/ssalehi/SynthScope/ARForensics hf_repo_id: Yanran21/ARForensics +train_batch_size: 32 # D3QE's training data (train/val splits) is fetched only when a train_transform_list is set +val_batch_size: 64 test_batch_size: 512 num_workers: 8 persistent_workers: true diff --git a/conf/dataset/image/mirage_test.yaml b/conf/dataset/image/mirage_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b4ea69f3e567ccf9dc972e4766bf7974a562149 --- /dev/null +++ b/conf/dataset/image/mirage_test.yaml @@ -0,0 +1,16 @@ +name: mirage_test +# Mirage-Test (OmniAID, arXiv 2511.08423) — real/fake images from recent realism-tuned generators in five categories +# (Human, Animal, Object, Scene, Anime; Anime is fakes only and is paired with the other categories' reals). +# (HF Yunncheng/Mirage-Test, MIT) +root_dir: /hddstore/ssalehi/SynthScope/MirageTest +hf_repo_id: Yunncheng/Mirage-Test +test_batch_size: 512 +num_workers: 8 +persistent_workers: true +pin_memory: true +prefetch: true +eval_transform_list: + - resize_short_256 + - center_crop_224 + - to_dtype + - normalize_clip diff --git a/conf/model/image/omniaid.yaml b/conf/model/image/omniaid.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bd8e84aa0e3da4c5a8df6d2bb0afe94dafdbdcd --- /dev/null +++ b/conf/model/image/omniaid.yaml @@ -0,0 +1,18 @@ +defaults: + - /model/default + - _self_ + +# OmniAID v2 (recommended; Mirage-Train, more robust augmentation). Expert configuration of the released checkpoint_omniaid_v2.pth (clean_test.py settings); +# read from the checkpoint's shapes (experts, rank, router width; top-k from the official config). +model_type: omniaid +CLIP_path: openai/clip-vit-large-patch14-336 +num_experts: 6 +rank_per_expert: 1 +moe_router_hidden_dim: 256 +moe_top_k: 2 +moe_lambda_orth: 0.001 +moe_lambda_balance: 0.001 +moe_lambda_gating_cls: 0.1 +dropout: 0.0 +gradient_checkpointing_enable: false +is_hybrid: true diff --git a/conf/model/image/omniaid_genimage.yaml b/conf/model/image/omniaid_genimage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a773fb5ebecca0b1966845bf7a9b15f2e37405f --- /dev/null +++ b/conf/model/image/omniaid_genimage.yaml @@ -0,0 +1,18 @@ +defaults: + - /model/default + - _self_ + +# OmniAID GenImage-SD v1.4 (the paper's GenImage/Mirage/AIGCDetect tables). Expert configuration of the released checkpoint_omniaid_genimage_sd14.pth (config_omniaid_genimage_paper.json); +# read from the checkpoint's shapes (experts, rank, router width; top-k from the official config). +model_type: omniaid +CLIP_path: openai/clip-vit-large-patch14-336 +num_experts: 3 +rank_per_expert: 4 +moe_router_hidden_dim: 64 +moe_top_k: 1 +moe_lambda_orth: 0.01 +moe_lambda_balance: 0.1 +moe_lambda_gating_cls: 0.1 +dropout: 0.0 +gradient_checkpointing_enable: false +is_hybrid: true diff --git a/conf/model/image/omniaid_v1.yaml b/conf/model/image/omniaid_v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b08fe3278398eab51e32715731c1bf9a3363b16 --- /dev/null +++ b/conf/model/image/omniaid_v1.yaml @@ -0,0 +1,18 @@ +defaults: + - /model/default + - _self_ + +# OmniAID v1 (Mirage-Train, the paper's OmniAID-Mirage). Expert configuration of the released checkpoint_omniaid_v1.pth (config_omniaid_mirage_paper.json); +# read from the checkpoint's shapes (experts, rank, router width; top-k from the official config). +model_type: omniaid +CLIP_path: openai/clip-vit-large-patch14-336 +num_experts: 6 +rank_per_expert: 8 +moe_router_hidden_dim: 256 +moe_top_k: 2 +moe_lambda_orth: 0.001 +moe_lambda_balance: 0.001 +moe_lambda_gating_cls: 0.1 +dropout: 0.0 +gradient_checkpointing_enable: false +is_hybrid: true diff --git a/conf/trainer/image/d3qe.yaml b/conf/trainer/image/d3qe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78c8a3a621491da3597301c77add8c53407a649a --- /dev/null +++ b/conf/trainer/image/d3qe.yaml @@ -0,0 +1,24 @@ +# D3QE's official training recipe (train.py + options.TrainOptions defaults; train.sh only sets the augmentation): +# AdamW lr 8e-5, betas (0.9, 0.999), weight decay 0.01, batch 32 on one GPU, 10 epochs, no warmup. Validation after +# every epoch keeps the best-accuracy checkpoint; EarlyStopping (patience 5, delta 0.001, then 0.002 after a drop) +# divides the learning rate by 10, stopping below 1e-6. The released checkpoint has total_steps 62500 = 10 epochs. +epochs: 10 +lr: 0.00008 +weight_decay: 0.01 +val_interval: 1 +test_interval: 1 +save_interval: 1 +save: true +save_policy: max +save_path: /ssdstore/ssalehi/SynthScope/models/d3qe +seed: 42 +warmup: + mode: "off" # quoted: a bare off is YAML boolean False, which the trainer reads as "on" + warmup_ratio: 0.0 + lr_scheduler: linear +plateau: + patience: 5 + delta: 0.001 + delta_after_drop: 0.002 + factor: 10 + min_lr: 0.000001 diff --git a/conf/trainer/image/npr.yaml b/conf/trainer/image/npr.yaml index 9da8b2d39204705876b095ed89e60c0dea6b38f5..5b8113c3e80b21cb6f8cca5b5b98118abddca045 100644 --- a/conf/trainer/image/npr.yaml +++ b/conf/trainer/image/npr.yaml @@ -2,17 +2,25 @@ defaults: - /trainer/default - _self_ +# NPR's official training recipe (README: train.py --classes car,cat,chair,horse --batch_size 32 --delr_freq 10 +# --lr 0.0002 --niter 50; networks/trainer.py: Adam, betas (0.9, 0.999), no weight decay): 4-class ProGAN, 50 epochs, +# learning rate x0.9 after epochs 10, 20, 30, 40; the released checkpoint is the last epoch's model. trainer_type: npr -epochs: 10 +epochs: 50 lr: 2.0e-4 -weight_decay: 0.0 +weight_decay: 0.0 # AdamW with weight decay 0 == the official Adam label_smoothing: 0.0 val_interval: 1 test_interval: 1 save_interval: 1 -save: false +save: true save_policy: max +save_path: /ssdstore/ssalehi/SynthScope/models/npr +seed: 100 # the official seed_torch(100) warmup: - mode: off - warmup_ratio: 0.05 + mode: "off" # quoted: a bare off is YAML boolean False, which the trainer reads as "on" + warmup_ratio: 0.0 lr_scheduler: linear +lr_step: + every: 10 + factor: 0.9 diff --git a/conf/trainer/image/pgc.yaml b/conf/trainer/image/pgc.yaml index 27ce08b698c59299c05a8ce8ad24dd695cb3fa10..4b54f76eed84b3b85c3580e855e2758474f22461 100644 --- a/conf/trainer/image/pgc.yaml +++ b/conf/trainer/image/pgc.yaml @@ -2,11 +2,15 @@ defaults: - /trainer/default - _self_ -# PGC trains LoRA adapters + the residual stream + PGCM + classifier (DINO backbone frozen). -# Official recipe (xiaoyu6868/PGC): AdamW, lr 5e-5, weight_decay 0.05, 100 epochs (--niter 100), -# CosineAnnealingLR over all steps, label smoothing 0.1, effective batch 128. +# PGC's official training recipe (xiaoyu6868/PGC train.py + engine/trainer.py, README 3.2): AdamW lr 5e-5, weight +# decay 0.05, betas (0.9, 0.999), label smoothing 0.1 (in the model's compute_loss), micro-batch 32 x 4 accumulation +# = 128, gradient-norm clipping 1.0, seed 42. The released ProGAN checkpoint is only ~400 steps in (the authors: +# DINOv2's prior overfits ProGAN within the first epoch), so the run is a fixed budget of micro-batches with a +# checkpoint every 100, and the early curve is evaluated. Official cosine schedule: T_max = 100 epochs of +# micro-batches but stepped once per optimizer step, so over the first 2000 micro-batches the LR stays at 5e-5 +# (>99.99%); it is run here at constant 5e-5. trainer_type: pgc -epochs: 100 +epochs: 1 lr: 5.0e-5 weight_decay: 0.05 label_smoothing: 0.1 @@ -15,7 +19,14 @@ test_interval: 1 save_interval: 1 save: true save_policy: max +save_path: /ssdstore/ssalehi/SynthScope/models/pgc +seed: 42 warmup: - mode: on - warmup_ratio: 0.05 + mode: "off" # quoted: a bare off is YAML boolean False + warmup_ratio: 0.0 lr_scheduler: cosine +step_budget: + max_steps: 2000 # micro-batches + save_every: 100 + grad_accum_steps: 4 + max_grad_norm: 1.0 diff --git a/dataset.py b/dataset.py index 5117b1a73ec785e9dbf7d69c9b73b4a36745671a..a4b3f13368ba64c19170f15345c55884acf0baf5 100644 --- a/dataset.py +++ b/dataset.py @@ -903,7 +903,8 @@ class CNNDetectionDataset(Dataset): "test": ("cnn_synth_testset", "test"), } - def __init__(self, root_dir: str | Path, split: Literal["train", "val", "test"], transform: AIGITransformations | None = None, prefetch: bool = False): + def __init__(self, root_dir: str | Path, split: Literal["train", "val", "test"], transform: AIGITransformations | None = None, prefetch: bool = False, + classes: list[str] | None = None): super().__init__() self.root_dir = Path(root_dir) self.split = split @@ -914,6 +915,12 @@ class CNNDetectionDataset(Dataset): print(f"Downloading {self.split} split from Google Drive...") self._download_data() self.images, self.labels, self.folder_names = self._load_data() + if classes: # e.g. the 4-class ProGAN setting (car, cat, chair, horse) of CNNDetection/NPR/FatFormer + keep = [i for i, f in enumerate(self.folder_names) if f in set(classes)] + if not keep: + raise ValueError(f"CNNDetection {split}: none of the classes {list(classes)} found") + self.images = [self.images[i] for i in keep]; self.labels = [self.labels[i] for i in keep] + self.folder_names = [self.folder_names[i] for i in keep] self.unique_folder_names = sorted(set(self.folder_names)) or [self.split] @classmethod @@ -1081,8 +1088,10 @@ class CNNDetectionDatasetModule: self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None prefetch = getattr(config.DatasetConfig, "prefetch", False) - self.train_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "train", self.train_transform, prefetch=prefetch) - self.val_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "val", self.eval_transform, prefetch=prefetch) + classes = config.DatasetConfig.get("train_classes", None) # ProGAN classes for train/val (None: all 20) + classes = list(classes) if classes else None + self.train_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "train", self.train_transform, prefetch=prefetch, classes=classes) + self.val_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "val", self.eval_transform, prefetch=prefetch, classes=classes) self.test_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch) def train_dataloader(self): @@ -2626,9 +2635,12 @@ class ARForensicsDataset(_TestOnlyImageDataset): Janus-Pro, RAR, Switti, Open-MAGVIT2), 6000 fakes and 6000 ImageNet reals each, laid out as ``ARForensics/test//{0_real,1_fake}``. Sourced from the HF dataset ``Yanran21/ARForensics`` (MIT): the corrected test release (MAR replaced by Switti) in four independent zips, ARForensics_updated_test_part00{1..4}.zip - (21 GB); a generator can span two zips, so all four are extracted into one folder.""" + (21 GB); a generator can span two zips, so all four are extracted into one folder. D3QE's training data is in the + same repo: ``train`` (100k LlamaGen fakes + 100k ImageNet reals; one zip split into 7 parts, 35 GB) and ``val`` + (10k + 10k, 3.7 GB), each laid out as ``ARForensics//{0_real,1_fake}``.""" hf_repo_id = "Yanran21/ARForensics" - hf_pattern = "ARForensics_updated_test_part*.zip" + hf_patterns = {"test": "ARForensics_updated_test_part*.zip", "val": "val/ARForensics_val.zip", + "train": "train/ARForensics_train_split.zip.a?"} @classmethod def prepare_split(cls, root_dir, split="test", force=False): @@ -2636,20 +2648,70 @@ class ARForensicsDataset(_TestOnlyImageDataset): if not force and _has_images(dest): return from huggingface_hub import snapshot_download - dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) - snapshot_download(repo_id=cls.hf_repo_id, repo_type="dataset", local_dir=str(dl), allow_patterns=[cls.hf_pattern]) - staging = root_dir / "_staging" - for archive in sorted(dl.glob(cls.hf_pattern)): + pattern = cls.hf_patterns[split] + dl = root_dir / f"_downloads_{split}"; dl.mkdir(parents=True, exist_ok=True) + snapshot_download(repo_id=cls.hf_repo_id, repo_type="dataset", local_dir=str(dl), allow_patterns=[pattern]) + archives = sorted(dl.glob(pattern)) + if split == "train": # a single zip split into parts: join them first + joined = dl / "train_joined.zip" + with open(joined, "wb") as out: + for part in archives: + with open(part, "rb") as f: + shutil.copyfileobj(f, out, 1 << 24) + part.unlink() + archives = [joined] + staging = root_dir / f"_staging_{split}" + for archive in archives: _safe_extract_zip(archive, staging) - inner = staging / "ARForensics" / "test" + inner = staging / "ARForensics" / split if not inner.is_dir(): - raise RuntimeError(f"ARForensics: expected ARForensics/test/ inside the archives, found {list(staging.iterdir())}") + raise RuntimeError(f"ARForensics: expected ARForensics/{split}/ inside the archives, found {list(staging.iterdir())}") if dest.exists(): shutil.rmtree(dest) shutil.move(str(inner), str(dest)) shutil.rmtree(dl, ignore_errors=True); shutil.rmtree(staging, ignore_errors=True) +class MirageTestDataset(_TestOnlyImageDataset): + """Mirage-Test (OmniAID, arXiv 2511.08423) — real and fake images from recent generators tuned for realism, in five + content categories laid out as ``/{0_real,1_fake}`` (Human, Animal, Object, Scene, Anime). Sourced from the + HF dataset ``Yunncheng/Mirage-Test`` (MIT). Anime ships fakes only ("due to copyright considerations") and the paper + scores it by its fake accuracy, so the loader pairs Anime with the reals of the other four categories: its + ``fake_acc`` is the paper's Anime number, and the other categories' balanced accuracy is the paper's "Overall".""" + hf_repo_id = "Yunncheng/Mirage-Test" + categories = ("Human", "Animal", "Object", "Scene", "Anime") + + @classmethod + def prepare_split(cls, root_dir, split="test", force=False): + root_dir = Path(root_dir); dest = root_dir / split + if not force and _has_images(dest): + return + from huggingface_hub import snapshot_download + dl = root_dir / "_downloads"; dl.mkdir(parents=True, exist_ok=True) + snapshot_download(repo_id=cls.hf_repo_id, repo_type="dataset", local_dir=str(dl), + allow_patterns=[f"{c}/*" for c in cls.categories]) + if dest.exists(): + shutil.rmtree(dest) + dest.mkdir(parents=True) + for c in cls.categories: + shutil.move(str(dl / c), str(dest / c)) + reals = sorted(p for c in cls.categories if c != "Anime" + for p in (dest / c / "0_real").rglob("*") if p.suffix.lower() in IMAGE_EXTENSIONS) + _link_images(reals, dest / "Anime" / "0_real") + shutil.rmtree(dl, ignore_errors=True) + + def _load_data(self): + # Fakes are nested one level deeper, by generator (``Human/1_fake//*.jpg``): the label comes from the + # 0_real/1_fake ancestor and the group from the category folder. + images, labels, folder_names = [], [], [] + for filepath in _iter_images_follow(self.destination_dir): + parts = filepath.relative_to(self.destination_dir).parts + images.append(filepath) + labels.append(1 if "1_fake" in parts else 0) + folder_names.append(parts[0]) + return images, labels, folder_names + + class _TestOnlyImageModule: """Shared loader for the eval-only image benchmarks (test split only).""" DATASET_CLASS = _TestOnlyImageDataset @@ -2724,8 +2786,36 @@ class RRDatasetModule(_TestOnlyImageModule): class ARForensicsDatasetModule(_TestOnlyImageModule): + """The test split for benchmarking; with a ``train_transform_list`` in the config also the train and val splits + (D3QE's training data), validated with ``val_transform_list`` (default: ``eval_transform_list``).""" DATASET_CLASS = ARForensicsDataset + def __init__(self, config): + super().__init__(config) + cfg = config.DatasetConfig + train_list = list(cfg.get("train_transform_list", [])) + self.train_dataset = self.val_dataset = None + if train_list: + prefetch = bool(cfg.get("prefetch", False)) + val_list = list(cfg.get("val_transform_list", cfg.get("eval_transform_list", []))) + self.train_dataset = ARForensicsDataset(cfg.root_dir, "train", AIGITransformations(train_list), prefetch=prefetch) + self.val_dataset = ARForensicsDataset(cfg.root_dir, "val", AIGITransformations(val_list), prefetch=prefetch) + + def _loader(self, dataset, batch_size, shuffle): + cfg = self.config.DatasetConfig + return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=cfg.num_workers, + persistent_workers=cfg.persistent_workers, pin_memory=cfg.pin_memory) + + def train_dataloader(self): + return None if self.train_dataset is None else self._loader(self.train_dataset, self.config.DatasetConfig.train_batch_size, True) + + def val_dataloader(self): + return None if self.val_dataset is None else self._loader(self.val_dataset, self.config.DatasetConfig.val_batch_size, False) + + +class MirageTestDatasetModule(_TestOnlyImageModule): + DATASET_CLASS = MirageTestDataset + class VideoFeedbackDataset(VideoFolderDataset): """VideoFeedback (TIGER-Lab) — used by STALL. 11 generators vs DiDeMo+Panda70M real, laid out @@ -2954,6 +3044,7 @@ SynthScopeModuleDict = { "bfree_online": BFreeOnlineDatasetModule, "rrdataset": RRDatasetModule, "arforensics": ARForensicsDatasetModule, + "mirage_test": MirageTestDatasetModule, # Video benchmarks. EvalCrafter/GenVidBench/VideoFeedback/ComGenVid have dedicated classes that # download + extract their HF archives into the real/fake layout; the rest use the generic # VideoFolderDatasetModule. diff --git a/deploy/hf-space/fetch_weights.py b/deploy/hf-space/fetch_weights.py index ec73228cccbae2365d053812e5250d0887d33132..e2496f1da55ee83b59c8b6947daa661d739cd6bf 100644 --- a/deploy/hf-space/fetch_weights.py +++ b/deploy/hf-space/fetch_weights.py @@ -15,8 +15,8 @@ sys.path.insert(0, str(ROOT)) # Detectors whose upstream licenses allow a free public demo. NPR's repository has no license; its first author gave # written permission (2026-10-11) to use NPR.pth in this free, non-commercial demo with credit (see LICENSES.md). -DEFAULT = ["clip", "cnn", "npr", "safe", "aide", "co_spy", "fat_former", "pgc", "cure", "dda", "d3qe", "vib_net", - "dinov2", "adapted_dino", "restrav"] +DEFAULT = ["clip", "cnn", "npr", "safe", "aide", "co_spy", "fat_former", "pgc", "cure", "dda", "d3qe", "omniaid", + "vib_net", "dinov2", "adapted_dino", "restrav"] def main() -> None: diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 5e33800d65f063f06cf61080d262b2ace0054a28..2e949714dd748d344c98c63320bde15618f7d765 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -86,6 +86,7 @@ have no automatic source: get them from their official sites and place them in t | `bfree_online` | BFree-Online (B-Free's viral images; used by DDA and CuRe): real and fake images that went viral, several web-scraped copies of each (about 1400 in the paper). Only URLs are released; the loader keeps an image only if its md5 matches, so dead links and placeholders are skipped (`/missing.txt`; in October 2026, 807 of the 1111 listed were still online: 269 real, 538 fake). One group, `test/viral` | `viral_images_dataset/BFree_viral_images.csv` in `grip-unina/B-Free` (nonprofit use only) | | `rrdataset` | RRDataset (ICCV 2025; used by CuRe): three scenarios, `original`, `transfer` (shared over online platforms) and `redigital` (re-digitized), about 8500 real and 8500 fake each. Only the test archive is used | Zenodo record `14963880` (`RRDataset_test.tar.gz`, 20 GB, CC BY 4.0) | | `arforensics` | ARForensics (ICCV 2025; used by D3QE): images from 7 visual autoregressive generators, LlamaGen, VAR, Infinity, Janus-Pro, RAR, Switti and Open-MAGVIT2, 6000 fakes and 6000 ImageNet reals each. The corrected test release (MAR replaced by Switti) | HF `Yanran21/ARForensics` (`ARForensics_updated_test_part00{1..4}.zip`, 21 GB, MIT) | +| `mirage_test` | Mirage-Test (OmniAID, arXiv 2511.08423): real and fake images from recent generators tuned for realism, in five categories: Human, Animal, Object, Scene and Anime. Anime is fakes only, so the loader pairs it with the other categories' reals; its fake accuracy is the paper's Anime number | HF `Yunncheng/Mirage-Test` (49k files, MIT) | | `diffusionforensics`, `diffusion1kstep`, `freqnetdetection` | DiffusionForensics, Diffusion-1kStep, FreqNet sets | see the yamls | | `genvideo` | GenVideo-Val (DeMamba; D3 Table 1, STALL): 10 generators vs 10,000 MSR-VTT reals, shared reals hard-linked into every generator | ModelScope `cccnju/Gen-Video` (`GenVideo-Val.zip`, 14 GB) | | `videophy` | VideoPhy test set (D3 Table 3): 3360 clips from 10 generators, released as URLs; dead links are skipped (both CogVideoX sets were offline in October 2026). Fake-only, so paired with GenVideo-Val's MSR-VTT reals, as D3 does | `videophysics/videophy_test_public` (MIT) + ModelScope for the reals | diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md index 6e3313d7b5ed94936420120908b7fbad8e70e966..9741c6eefe6ae551d71651eecd431362cb0d736f 100644 --- a/docs/CAPABILITIES.md +++ b/docs/CAPABILITIES.md @@ -30,6 +30,7 @@ matching `conf/model//.yaml`, a `trainer.py._build_model` branch, | `pgc` | image | bce | official | pgc_peak | LoRA-DINOv2 + YCbCr residual, peak-guided | | `cure` | image | ce | official | – | Frozen PE-Core L/14 + Q/V LoRA + source-response subspace (mixing-ratio regression) | | `d3qe` | image | bce | official | – | LlamaGen VQ quantization error + codebook-aware transformer + CLIP ViT-L/14 (autoregressive images) | +| `omniaid` (+ `omniaid_v1`, `omniaid_genimage`) | image | ce | official | – | Hybrid MoE on CLIP ViT-L/14@336: routed semantic experts + an always-on artifact expert | | `dda` | image | bce | official | – | DINOv2 ViT-L/14 + LoRA, pixel- and frequency-aligned training data (center-crop 336) | | `vib_net` | image | vib | synthscope | – | Variational information bottleneck (FLAVA) | | `d3` | video | video | training-free | – | Disentangled detail dynamics (X-CLIP) | diff --git a/docs/WILD_BENCHMARKS.md b/docs/WILD_BENCHMARKS.md index acadf4c5c47b2d165c9317fabcf5f529550757e8..64e57dd0734ee1c076bcee58a2d4243b5ce63376 100644 --- a/docs/WILD_BENCHMARKS.md +++ b/docs/WILD_BENCHMARKS.md @@ -14,6 +14,7 @@ Each number comes from `repro/bench_results/_.json`; regenerate | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | CuRe | **79.7 / 85.5** | **98.1 / 99.7** | **92.7 / 96.5** | **95.9 / 99.1** | **91.6 / 98.1** | **82.0 / 88.9** | 90.0 | | DDA | 67.2 / 75.0 | **97.9 / 99.8** | **90.8 / 96.4** | **90.2 / 96.1** | **95.0 / 98.9** | **76.0 / 81.7** | 86.2 | +| OmniAID (v2) | 61.7 / 83.6 | 74.1 / 93.1 | **86.0 / 95.3** | **88.4 / 96.9** | 70.3 / 87.2 | 71.8 / 82.3 | 75.4 | | SPAI | 61.6 / 69.0 | **93.8 / 98.7** | **76.2 / 84.6** | **80.6 / 88.4** | 59.8 / 60.2 | 64.8 / 72.0 | 72.8 | | Co-Spy | 71.0 / 76.5 | **96.7 / 99.5** | 64.3 / 69.5 | 61.6 / 71.2 | 56.3 / 73.3 | 66.8 / 73.9 | 69.4 | | VIB-Net | 65.8 / 74.3 | **94.8 / 99.7** | 58.5 / 73.6 | 56.1 / 71.3 | 51.9 / 53.6 | 60.0 / 66.4 | 64.5 | diff --git a/experiments/image/d3qe.sh b/experiments/image/d3qe.sh new file mode 100644 index 0000000000000000000000000000000000000000..06a6c4009b9108c5ad3cbd8851fca331a5199188 --- /dev/null +++ b/experiments/image/d3qe.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# D3QE (ICCV 2025): the official training recipe, reproduced in SynthScope. +# data ARForensics train (100k LlamaGen vs 100k ImageNet) and val (10k + 10k); downloaded on first use +# model frozen LlamaGen VQ-16 + frozen CLIP ViT-L/14; only the D3AT transformer and the heads are trained +# train short side 256 -> CNNDetection blur/JPEG (p 0.1 each) -> RandomCrop 256 -> flip -> [0, 1], no normalization +# val short side 256 -> the same blur/JPEG at fixed settings (sigma 1.5, PIL q65) -> CenterCrop 256 +# optimizer AdamW lr 8e-5, weight decay 0.01, batch 32 on ONE GPU (BatchNorm in the heads), 10 epochs, +# best-validation-accuracy checkpoint, learning rate /10 after 5 epochs without improvement +# Evaluate the result with: python benchmark/bench_eval.py --model d3qe --ckpt /model_best.pth --dataset arforensics +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/nvmestore/ssalehi/.triton} +export HF_HOME=${HF_HOME:-/nvmestore/ssalehi/.cache/huggingface} +export TORCH_HOME=${TORCH_HOME:-/nvmestore/ssalehi/.cache/torch} +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} +export OMP_NUM_THREADS=1 +DATA=${DATA:-/nvmestore/ssalehi/SynthScope/ARForensics} +SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/d3qe} + +# Plain single-process run (no `accelerate launch`): the repo's accelerate_config.yaml uses DeepSpeed with gradient +# clipping 1.0, which the official recipe does not have. +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope python main.py \ + trainer@TrainerConfig=image/d3qe \ + model@ModelConfig=image/d3qe \ + dataset@DatasetConfig=image/arforensics \ + DatasetConfig.root_dir=$DATA \ + DatasetConfig.train_batch_size=32 \ + DatasetConfig.val_batch_size=64 \ + "+DatasetConfig.train_transform_list=[resize_short_256,blur_jpeg_cnndet,random_crop_256,random_horizontal_flip,to_dtype]" \ + "+DatasetConfig.val_transform_list=[resize_short_256,blur_jpeg_cnndet_val,center_crop_256,to_dtype]" \ + "DatasetConfig.eval_transform_list=[resize_short_256,center_crop_256,to_dtype]" \ + TrainerConfig.save_path=$SAVE \ + "metrics=[real_accuracy,fake_accuracy]" \ + logger.name=d3qe \ + logger.mode=offline \ + "logger.tags=[SynthScope,D3QE,arforensics]" diff --git a/experiments/image/npr.sh b/experiments/image/npr.sh index 1fc9a15148981bc0581f146eb449814dfd36e2ad..2186799ed289eb040a56cbe322227c6d1afb3fd9 100644 --- a/experiments/image/npr.sh +++ b/experiments/image/npr.sh @@ -1,20 +1,34 @@ #!/bin/bash -export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton -export HF_HOME=/nvmestore/ssalehi/.cache/huggingface -export TORCH_HOME=/nvmestore/ssalehi/.cache/torch -export CUDA_VISIBLE_DEVICES=1,2 +# NPR (CVPR 2024): the official training recipe, reproduced in SynthScope. +# data ForenSynths / CNNDetection ProGAN train + val, 4-class setting (car, cat, chair, horse) +# train Resize((256, 256)) -> RandomCrop 224 -> flip -> ImageNet normalization (no blur/JPEG: the official +# data_augment call is commented out); NPR's neighboring-pixel residual is computed inside the model +# val Resize((256, 256)) -> CenterCrop 224 -> ImageNet normalization +# optimizer Adam lr 2e-4 (AdamW, weight decay 0), batch 32 on ONE GPU, 50 epochs, lr x0.9 every 10 epochs +# checkpoint model_last.pth (the official code releases the last epoch); model_best.pth is kept for reference +# Evaluate with: python benchmark/bench_eval.py --model npr --ckpt /model_last.pth --benchmark forensynths --data +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/nvmestore/ssalehi/.triton} +export HF_HOME=${HF_HOME:-/nvmestore/ssalehi/.cache/huggingface} +export TORCH_HOME=${TORCH_HOME:-/nvmestore/ssalehi/.cache/torch} +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} export OMP_NUM_THREADS=1 +DATA=${DATA:-/nvmestore/ssalehi/SynthScope/CNNDetection} +SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/npr} -# NPR trains a small ResNet from scratch on neighboring-pixel residuals; feed [0,1] images (to_dtype only). -/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ +# Plain single-process run (no `accelerate launch`): the repo's accelerate_config.yaml uses DeepSpeed with gradient +# clipping 1.0, which the official recipe does not have. +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope python main.py \ trainer@TrainerConfig=image/npr \ model@ModelConfig=image/npr \ - dataset@DatasetConfig=image/genimage \ - DatasetConfig.train_batch_size=128 \ - DatasetConfig.val_batch_size=512 \ - DatasetConfig.test_batch_size=512 \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype] \ + dataset@DatasetConfig=image/cnndetection \ + DatasetConfig.root_dir=$DATA \ + "+DatasetConfig.train_classes=[car,cat,chair,horse]" \ + DatasetConfig.train_batch_size=32 \ + DatasetConfig.val_batch_size=64 \ + "DatasetConfig.train_transform_list=[resize_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" \ + "DatasetConfig.eval_transform_list=[resize_256,center_crop_224,to_dtype,normalize_dinov2]" \ + TrainerConfig.save_path=$SAVE \ + "metrics=[real_accuracy,fake_accuracy]" \ logger.name=npr \ - logger.mode=online \ - logger.tags=[SynthScope,NPR,genimage] + logger.mode=offline \ + "logger.tags=[SynthScope,NPR,progan4]" diff --git a/experiments/image/pgc.sh b/experiments/image/pgc.sh index 3f2e23d000d9c2e4a29372e3ac750d11602daecf..7f614de3bad478bf159c3c205d79da2c43049ab4 100644 --- a/experiments/image/pgc.sh +++ b/experiments/image/pgc.sh @@ -1,27 +1,34 @@ #!/bin/bash -export TRITON_CACHE_DIR=/nvmestore/ssalehi/.triton -export HF_HOME=/nvmestore/ssalehi/.cache/huggingface -export TORCH_HOME=/nvmestore/ssalehi/.cache/torch -export CUDA_VISIBLE_DEVICES=1,2 +# PGC (Zhou et al., 2026): the official ProGAN training recipe, reproduced in SynthScope. +# data ForenSynths / CNNDetection ProGAN train, all 20 classes (720k images = the UniversalFakeDetect train set) +# train flip -> ColorJitter(0.15, 0.15, 0.15, 0.05) p0.2 -> pad + RandomCrop 224 (no resize: the YCbCr residual +# is resolution-sensitive) -> ImageNet normalization; the residual is computed inside the model +# optimizer AdamW lr 5e-5 (constant over this budget), weight decay 0.05, label smoothing 0.1, batch 32 x 4 +# accumulation on ONE GPU, gradient clipping 1.0, seed 42 +# budget 2000 micro-batches with a checkpoint every 100 (model_step.pth): the released checkpoint is ~400 steps +# in and transfer to unseen generators erodes within the first epoch, so the early curve is evaluated +# Evaluate: python benchmark/bench_eval.py --model pgc --ckpt /model_step.pth --benchmark forensynths --data +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/nvmestore/ssalehi/.triton} +export HF_HOME=${HF_HOME:-/nvmestore/ssalehi/.cache/huggingface} +export TORCH_HOME=${TORCH_HOME:-/nvmestore/ssalehi/.cache/torch} +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} export OMP_NUM_THREADS=1 +DATA=${DATA:-/nvmestore/ssalehi/SynthScope/CNNDetection} +SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/pgc} -# PGC (Zhou et al., 2026): LoRA-tuned DINOv2-Large RGB stream + YCbCr-quantization-residual CNN -# stream, fused by peak-guided calibration. The residual is computed INSIDE the model, so the -# dataset feeds a normal ImageNet-normalized image (normalize_dinov2 = what DINOv2 expects). The -# residual is resolution-sensitive, so we use CROP-only transforms (no resize) to preserve the -# per-pixel quantization pattern. Trained here on ProGAN (CNNDetection); swap the dataset for -# GenImage to match the paper's SDv1.4 setting. -/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ +# Plain single-process run (no `accelerate launch`): gradient accumulation and clipping are done by the trainer's +# step budget exactly as in the official code. +/home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope python main.py \ trainer@TrainerConfig=image/pgc \ model@ModelConfig=image/pgc \ dataset@DatasetConfig=image/cnndetection \ - ModelConfig.backbone_name=facebook/dinov2-large \ + DatasetConfig.root_dir=$DATA \ DatasetConfig.train_batch_size=32 \ DatasetConfig.val_batch_size=64 \ - DatasetConfig.test_batch_size=64 \ - DatasetConfig.train_transform_list=[random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ - DatasetConfig.eval_transform_list=[center_crop_224,to_dtype,normalize_dinov2] \ - metrics=[real_accuracy,fake_accuracy,AP,AUC] \ + "DatasetConfig.train_transform_list=[random_horizontal_flip,color_jitter_pgc,pad_random_crop_224,to_dtype,normalize_dinov2]" \ + "DatasetConfig.eval_transform_list=[center_crop_224,to_dtype,normalize_dinov2]" \ + TrainerConfig.save_path=$SAVE \ + "metrics=[real_accuracy,fake_accuracy]" \ logger.name=pgc \ - logger.mode=online \ - logger.tags=[SynthScope,PGC,cnndetection] + logger.mode=offline \ + "logger.tags=[SynthScope,PGC,progan20]" diff --git a/interface/backend/build_repro_full.py b/interface/backend/build_repro_full.py index 907141d9635c572f73862eec0adbcea29ba85710..6b4cd28e8d8ac3dfe16cb8fcbd027106312778a0 100644 --- a/interface/backend/build_repro_full.py +++ b/interface/backend/build_repro_full.py @@ -50,21 +50,37 @@ VAL = { ("dda", "RealChain"): {"official": {"map": 76.46, "acc": 67.23, "auroc": 74.97}}, ("dda", "RRDataset"): {"official": {"map": 85.18, "acc": 75.99, "auroc": 81.74}}, # D3QE: only the official checkpoint is evaluated (no SynthScope-trained number). acc = plain = balanced (balanced subsets). - ("d3qe", "ARForensics"): {"official": {"map": 92.22, "acc": 81.74, "auroc": 91.91}}, - ("d3qe", "ForenSynths"): {"official": {"map": 92.23, "acc": 83.45, "auroc": 90.65}}, - ("d3qe", "GenImage"): {"official": {"map": 90.94, "acc": 79.44, "auroc": 90.49}}, + # synthscope = trained in SynthScope with the official recipe (experiments/image/d3qe.sh; job 265064, eval 265826). + ("d3qe", "ARForensics"): {"official": {"map": 92.22, "acc": 81.74, "auroc": 91.91}, "synthscope": {"map": 91.73, "acc": 79.83, "auroc": 91.49}}, + ("d3qe", "ForenSynths"): {"official": {"map": 92.23, "acc": 83.45, "auroc": 90.65}, "synthscope": {"map": 91.35, "acc": 82.14, "auroc": 90.29}}, + ("d3qe", "GenImage"): {"official": {"map": 90.94, "acc": 79.44, "auroc": 90.49}, "synthscope": {"map": 90.11, "acc": 77.01, "auroc": 89.63}}, ("d3qe", "SynthWildX"): {"official": {"map": 49.3, "acc": 49.58, "auroc": 45.47}}, ("d3qe", "WildRF"): {"official": {"map": 63.65, "acc": 53.03, "auroc": 63.89}}, ("d3qe", "BFree-Online"): {"official": {"map": 68.77, "acc": 47.03, "auroc": 64.0}}, ("d3qe", "RealChain"): {"official": {"map": 47.24, "acc": 49.49, "auroc": 43.49}}, ("d3qe", "EvalGEN"): {"official": {"map": 91.48, "acc": 62.21, "auroc": 83.94}}, ("d3qe", "RRDataset"): {"official": {"map": 59.25, "acc": 54.95, "auroc": 55.11}}, + # OmniAID: official checkpoints only. Mirage-Test acc = the paper's metric (4 balanced categories + Anime fake accuracy). + ("omniaid", "GenImage (8 subsets) — GenImage-SD1.4"): {"official": {"map": 99.02, "acc": 95.82, "auroc": 99.16}}, + ("omniaid", "AIGCDetectBenchmark (17) — GenImage-SD1.4"): {"official": {"map": 97.88, "acc": 88.87, "auroc": 98.2}}, + ("omniaid", "Mirage-Test (5 categories) — GenImage-SD1.4"): {"official": {"acc": 51.1}}, + ("omniaid", "GenImage (8 subsets) — Mirage-Train v1"): {"official": {"map": 99.2, "acc": 97.16, "auroc": 99.12}}, + ("omniaid", "AIGCDetectBenchmark (17) — Mirage-Train v1"): {"official": {"map": 98.93, "acc": 92.88, "auroc": 98.99}}, + ("omniaid", "Mirage-Test (5 categories) — Mirage-Train v1"): {"official": {"acc": 88.39}}, + ("omniaid", "Mirage-Test (v2"): {"official": {"map": 96.48, "acc": 78.46, "auroc": 96.65}}, + ("omniaid", "SynthWildX (v2"): {"official": {"map": 95.52, "acc": 86.02, "auroc": 95.31}}, + ("omniaid", "WildRF (v2"): {"official": {"map": 97.28, "acc": 88.36, "auroc": 96.91}}, + ("omniaid", "BFree-Online (v2"): {"official": {"map": 91.7, "acc": 70.26, "auroc": 87.18}}, + ("omniaid", "RealChain (v2"): {"official": {"map": 84.02, "acc": 61.71, "auroc": 83.63}}, + ("omniaid", "EvalGEN (v2"): {"official": {"map": 96.46, "acc": 74.14, "auroc": 93.08}}, + ("omniaid", "RRDataset (v2"): {"official": {"map": 84.19, "acc": 71.75, "auroc": 82.32}}, ("fat_former", "4-class"): {"official": {"map": 99.7, "acc": 98.39}}, ("fat_former", "Diffusion-model"): {"official": {"map": 98.6, "acc": 93.72}}, - ("npr", "Table 1"): {"official": {"map": 94.22, "acc": 92.30}}, + # synthscope = trained in SynthScope with the official recipe (experiments/image/npr.sh; job 265720, eval 265808). + ("npr", "Table 1"): {"official": {"map": 94.22, "acc": 92.30}, "synthscope": {"map": 93.83, "acc": 84.09, "auroc": 94.94}}, ("npr", "Table 2"): {"official": {"map": 96.49, "acc": 94.81}}, ("npr", "Table 3"): {"official": {"map": 99.4, "acc": 95.03}}, - ("npr", "Table 4"): {"official": {"map": 96.69, "acc": 93.94}}, + ("npr", "Table 4"): {"official": {"map": 96.69, "acc": 93.94}, "synthscope": {"map": 97.49, "acc": 88.91, "auroc": 97.23}}, ("npr", "Table 5"): {"official": {"map": 83.47, "acc": 79.13}}, # SAFE now SERVES the SynthScope-trained checkpoint (retrain v2 / model_38), which reproduces the paper. ("safe", "ForenSynths"): {"official": {"map": 98.78, "acc": 96.2}, "synthscope": {"map": 97.79, "acc": 95.1}}, @@ -74,8 +90,9 @@ VAL = { ("safe", "DiTFake"): {"synthscope": {"map": 99.96, "acc": 99.2}}, ("safe", "DiffusionForensics"): {"synthscope": {"map": 99.34, "acc": 91.3}}, ("pgc", "GenImage"): {"official": {"map": 100.0, "acc": 99.49}, "synthscope": {"map": 97.62, "acc": 78.6}}, - ("pgc", "UniversalFakeDetect"): {"official": {"map": 99.34, "acc": 94.16}, "synthscope": {"map": 99.01}}, - ("pgc", "ForenSynths"): {"official": {"map": 96.25, "acc": 83.73}, "synthscope": {"map": 89.03}}, + # synthscope UFD / ForenSynths = the official recipe in SynthScope, step 500 (experiments/image/pgc.sh; jobs 266276, 266282, 266660). + ("pgc", "UniversalFakeDetect"): {"official": {"map": 99.34, "acc": 94.16}, "synthscope": {"map": 98.99, "acc": 91.96, "auroc": 98.92}}, + ("pgc", "ForenSynths"): {"official": {"map": 96.25, "acc": 83.73}, "synthscope": {"map": 97.41, "acc": 85.89, "auroc": 97.63}}, ("pgc", "AIGIBench"): {"official": {"map": 95.36, "acc": 88.33}, "synthscope": {"map": 86.88}}, ("pgc", "CommGen15"): {"official": {"map": 99.26, "acc": 97.39}, "synthscope": {"map": 98.61, "acc": 66.66}}, # SPAI now SERVES official weights remapped into the SynthScope class, scored at NATIVE resolution diff --git a/interface/backend/datasets.py b/interface/backend/datasets.py index 00f252665080882a3a056062fd05d8b251e3cc42..be1c81476ccd6fa633474211df83fcf65496d4b3 100644 --- a/interface/backend/datasets.py +++ b/interface/backend/datasets.py @@ -120,6 +120,9 @@ DATASET_DEFS: list[DatasetDef] = [ DatasetDef("rrdataset", "RRDataset (original / transfer / redigital)", "image", ["/hddstore/ssalehi/SynthScope/RRDataset"], ["test"], "Real vs AI images as created, after sharing online, and after re-digitization (ICCV 2025)."), + DatasetDef("mirage_test", "Mirage-Test (human, animal, object, scene, anime)", "image", + ["/hddstore/ssalehi/SynthScope/MirageTest"], ["test"], + "Recent generators tuned for realism, in five content categories (OmniAID); Anime is fakes only."), DatasetDef("arforensics", "ARForensics (7 autoregressive generators)", "image", ["/hddstore/ssalehi/SynthScope/ARForensics"], ["test"], "LlamaGen, VAR, Infinity, Janus-Pro, RAR, Switti and Open-MAGVIT2 vs ImageNet reals, 6000 each (D3QE, ICCV 2025)."), diff --git a/interface/backend/engine.py b/interface/backend/engine.py index b3def357a12746020c04135bab35c8a009706a05..8ddc1c6f7770080db2a4d769509c94873f89f961 100644 --- a/interface/backend/engine.py +++ b/interface/backend/engine.py @@ -107,6 +107,9 @@ def _build_model(cfg): if model_type == "dda": from models.image.dda.dda import SynthScopeDDAModel return SynthScopeDDAModel(cfg) + if model_type == "omniaid": + from models.image.omniaid.omniaid import SynthScopeOmniAIDModel + return SynthScopeOmniAIDModel(cfg) if model_type == "d3qe": from models.image.d3qe.d3qe import SynthScopeD3QEModel return SynthScopeD3QEModel(cfg) diff --git a/interface/backend/registry.py b/interface/backend/registry.py index b02213b51de3dee58bd08f7a106eea0ac1283bf1..137f87b5a91e1ae5dd35405faa13b8e10d99c6ed 100644 --- a/interface/backend/registry.py +++ b/interface/backend/registry.py @@ -347,6 +347,26 @@ _add(ModelSpec( "tokenizer, read by a small transformer, fused with frozen CLIP ViT-L/14 features. Trained on LlamaGen vs " "ImageNet only. Uses the OFFICIAL released checkpoint.", )) +# OmniAID's official eval preprocessing: v2 resizes straight to 336 (data/datasets.py); the v1-era releases used a +# double resize (512x512, then 336x336) - the commented-out pipeline that the official code says to use for v1 weights. +omniaid_v1_transform = T.Compose([T.Resize((512, 512)), T.Resize((336, 336)), T.ToTensor(), _norm(CLIP_MEAN, CLIP_STD)]) +for _key, _name, _ckpt, _tf, _note in [ + ("omniaid", "OmniAID", "omniaid.pth", tf_square_resize(336, None, CLIP_MEAN, CLIP_STD), + "the recommended v2 release, trained on Mirage-Train (the authors' modern in-the-wild dataset)"), + ("omniaid_v1", "OmniAID (v1)", "omniaid_v1.pth", omniaid_v1_transform, + "the v1 release trained on Mirage-Train, the paper's OmniAID-Mirage"), + ("omniaid_genimage", "OmniAID (GenImage)", "omniaid_genimage.pth", omniaid_v1_transform, + "the release trained on GenImage SD v1.4 only, for the paper's academic GenImage protocol"), +]: + _add(ModelSpec( + key=_key, name=_name, paper="Guo et al., arXiv 2511.08423", + modality="image", predict_kind="ce", checkpoint=_ckpt, ckpt_kind="official_omniaid", weights="official", + # .pth = the official ckpt/checkpoint_omniaid_*.pth unchanged (HF Yunncheng/OmniAID); full model, strict load. + overrides=[f"model@ModelConfig=image/{_key}"], + transform=_tf, + notes="Hybrid mixture of experts on CLIP ViT-L/14@336: a router picks semantic experts (human, animal, object, " + "scene, ...) and an always-on artifact expert sees every image. Uses the OFFICIAL checkpoint: " + _note + ".", + )) _add(ModelSpec( key="npr", name="NPR", paper="Tan et al., CVPR'24", modality="image", predict_kind="bce_npr", checkpoint="npr.pth", @@ -417,6 +437,9 @@ INPUT_NOTES: dict[str, tuple[str, str]] = { "images are padded with black; the rest of a larger image is not seen."), "d3qe": ("Resized to 256, center crop", "The shorter side is scaled to 256 and the central 256×256 square is used; " "its CLIP branch looks at the central 224×224 of that square."), + "omniaid": ("Squashed to 336×336", "The whole image is stretched to a 336×336 square (the aspect ratio changes)."), + "omniaid_v1": ("Squashed to 512, then 336", "The image is stretched to 512×512 and then to 336×336, as in the v1 release."), + "omniaid_genimage": ("Squashed to 512, then 336", "The image is stretched to 512×512 and then to 336×336, as in the v1-era releases."), "cure": ("Paper-specific pipeline", "As in the paper: PNG images are re-saved as JPEG, the shorter side is scaled to " "390 and center-cropped, resized to 336, then JPEG-compressed again (quality 70)."), } diff --git a/interface/backend/repro_full.json b/interface/backend/repro_full.json index 12c306c6ad8e3bdc59a913cbdd563f545fc11752..89e4b8fd7d9ad4a4b6232dfdcc7f7dd24a3aa421 100644 --- a/interface/backend/repro_full.json +++ b/interface/backend/repro_full.json @@ -286,8 +286,8 @@ "benchmark": "ARForensics (7 autoregressive generators, full test set)", "metric": "accuracy / AP (mean over generators)", "paper": "acc=82.11 / map=92.07", - "ours": "mAP=92.22 / acc=81.74 / AUROC=91.91 (official wts)", - "ours_synthscope": "", + "ours": "SynthScope mAP=91.73 / acc=79.83 / AUROC=91.49 \u00b7 official mAP=92.22 / acc=81.74 / AUROC=91.91", + "ours_synthscope": "mAP=91.73 / acc=79.83 / AUROC=91.49", "ours_official": "mAP=92.22 / acc=81.74 / AUROC=91.91", "status": "official", "note": "Official checkpoint through SynthScope: 81.74 acc / 92.22 AP (AUROC 91.91) on all 84,000 images; LlamaGen 97.16/99.58, VAR 84.46/95.16, Infinity 63.06/80.26, Janus-Pro 92.39/97.85, RAR 91.70/97.88, Switti 74.23/88.71, Open-MAGVIT2 69.21/86.12, every generator within 1.1 pt acc (tag d3qe_arforensics, job 264795)." @@ -296,20 +296,20 @@ "benchmark": "ForenSynths (paper's 7 GANs, cross-paradigm)", "metric": "accuracy / AP (mean over generators)", "paper": "acc=83.73 / map=92.23", - "ours": "mAP=92.23 / acc=83.45 / AUROC=90.65 (official wts)", - "ours_synthscope": "", + "ours": "SynthScope mAP=91.35 / acc=82.14 / AUROC=90.29 \u00b7 official mAP=92.23 / acc=83.45 / AUROC=90.65", + "ours_synthscope": "mAP=91.35 / acc=82.14 / AUROC=90.29", "ours_official": "mAP=92.23 / acc=83.45 / AUROC=90.65", - "status": "official", + "status": "reproduced", "note": "Official checkpoint through SynthScope (bench_eval --dataset cnndetection --select the 7 GANs): 83.45 acc / 92.23 AP (AUROC 90.65); ProGAN 95.10, StyleGAN 76.21, StyleGAN2 75.08, BigGAN 86.07, CycleGAN 83.27, StarGAN 73.64, GauGAN 94.76, every GAN within 1.5 pt (tag d3qe_cnndetection, job 264795)." }, { "benchmark": "GenImage (paper's 6 diffusion subsets, cross-paradigm)", "metric": "accuracy / AP (mean over generators)", "paper": "acc=78.61 / map=89.6", - "ours": "mAP=90.94 / acc=79.44 / AUROC=90.49 (official wts)", - "ours_synthscope": "", + "ours": "SynthScope mAP=90.11 / acc=77.01 / AUROC=89.63 \u00b7 official mAP=90.94 / acc=79.44 / AUROC=90.49", + "ours_synthscope": "mAP=90.11 / acc=77.01 / AUROC=89.63", "ours_official": "mAP=90.94 / acc=79.44 / AUROC=90.49", - "status": "official", + "status": "reproduced", "note": "Official checkpoint through SynthScope (bench_eval --dataset genimage_full --select the 6 subsets): 79.44 acc / 90.94 AP (AUROC 90.49), the mean within 0.9 pt; per subset the gaps are larger (ADM 63.67, -6.8; GLIDE 91.13; Midjourney 63.80; SD-v1.4 82.21; SD-v1.5 85.10; Wukong 90.73, +6.3), so the paper's GenImage test sample probably differs from ours (tag d3qe_genimage_full, job 264795)." }, { @@ -542,8 +542,8 @@ "benchmark": "Table 1 \u2014 ForenSynths / Cross-GAN-Sources (8 GANs) \u2014 HEADLINE", "metric": "AP (and acc@0.5)", "paper": "map=96.1 / acc=92.5", - "ours": "mAP=94.22 / acc=92.3 (official wts)", - "ours_synthscope": "", + "ours": "SynthScope mAP=93.83 / acc=84.09 / AUROC=94.94 \u00b7 official mAP=94.22 / acc=92.3", + "ours_synthscope": "mAP=93.83 / acc=84.09 / AUROC=94.94", "ours_official": "mAP=94.22 / acc=92.3", "status": "official", "note": "Our npr_official_forensynths.json logs the full 13-gen mean 79.45 mAP; the 8-GAN subset {progan,cyclegan,biggan,gaugan,stargan,stylegan,stylegan2,deepfake} computes to 92.81 mAP / 91.83 acc." @@ -572,8 +572,8 @@ "benchmark": "Table 4 \u2014 UniversalFakeDetect / Ojha", "metric": "AP (and acc@0.5)", "paper": "acc=95.2", - "ours": "mAP=96.69 / acc=93.94 (official wts)", - "ours_synthscope": "", + "ours": "SynthScope mAP=97.49 / acc=88.91 / AUROC=97.23 \u00b7 official mAP=96.69 / acc=93.94", + "ours_synthscope": "mAP=97.49 / acc=88.91 / AUROC=97.23", "ours_official": "mAP=96.69 / acc=93.94", "status": "official", "note": "n/a \u2014 96.69 mAP / 93.94 acc, close to paper acc 95.2." @@ -589,6 +589,148 @@ "note": "n/a \u2014 83.47 mAP / 79.13 acc \u2248 paper acc 80.1." } ], + "omniaid": [ + { + "benchmark": "GenImage (8 subsets) \u2014 GenImage-SD1.4 release", + "metric": "accuracy (mean over subsets)", + "paper": "acc=95.94", + "ours": "mAP=99.02 / acc=95.82 / AUROC=99.16 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=99.02 / acc=95.82 / AUROC=99.16", + "status": "official", + "note": "omniaid_genimage through SynthScope: 95.82 (mAP 99.02), every subset within 1.5 pt; v1-era 512->336 preprocessing (tag omniaid_genimage_genimage_full, job 265856)." + }, + { + "benchmark": "AIGCDetectBenchmark (17) \u2014 GenImage-SD1.4 release", + "metric": "accuracy (mean over subsets)", + "paper": "acc=88.87", + "ours": "mAP=97.88 / acc=88.87 / AUROC=98.2 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=97.88 / acc=88.87 / AUROC=98.2", + "status": "official", + "note": "omniaid_genimage through SynthScope: 88.87 (mAP 97.88), exactly the paper's number (tag omniaid_genimage_aigcdetect, job 265856)." + }, + { + "benchmark": "Mirage-Test (5 categories) \u2014 GenImage-SD1.4 release", + "metric": "accuracy (mean of Human/Animal/Object/Scene balanced + Anime fake accuracy)", + "paper": "acc=51.1", + "ours": "acc=51.1 (official wts)", + "ours_synthscope": "", + "ours_official": "acc=51.1", + "status": "official", + "note": "omniaid_genimage through SynthScope: 51.10, every category equal to the paper (Scene 46.52) (tag omniaid_genimage_mirage_test, job 265935)." + }, + { + "benchmark": "GenImage (8 subsets) \u2014 Mirage-Train v1 release (OmniAID-Mirage)", + "metric": "accuracy (mean over subsets)", + "paper": "acc=97.24", + "ours": "mAP=99.2 / acc=97.16 / AUROC=99.12 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=99.2 / acc=97.16 / AUROC=99.12", + "status": "official", + "note": "omniaid_v1 through SynthScope: 97.16 (mAP 99.20), every subset within 0.6 pt (tag omniaid_v1_genimage_full, job 265857)." + }, + { + "benchmark": "AIGCDetectBenchmark (17) \u2014 Mirage-Train v1 release (OmniAID-Mirage)", + "metric": "accuracy (mean over subsets)", + "paper": "acc=92.88", + "ours": "mAP=98.93 / acc=92.88 / AUROC=98.99 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=98.93 / acc=92.88 / AUROC=98.99", + "status": "official", + "note": "omniaid_v1 through SynthScope: 92.88 (mAP 98.93), exactly the paper's number (tag omniaid_v1_aigcdetect, job 265857)." + }, + { + "benchmark": "Mirage-Test (5 categories) \u2014 Mirage-Train v1 release (OmniAID-Mirage)", + "metric": "accuracy (mean of Human/Animal/Object/Scene balanced + Anime fake accuracy)", + "paper": "acc=88.39", + "ours": "acc=88.39 (official wts)", + "ours_synthscope": "", + "ours_official": "acc=88.39", + "status": "official", + "note": "omniaid_v1 through SynthScope: 88.39, every category equal to the paper (tag omniaid_v1_mirage_test, job 265936)." + }, + { + "benchmark": "Chameleon / DRCT-2M", + "metric": "accuracy", + "paper": "acc=77.35", + "ours": "", + "ours_synthscope": "", + "ours_official": "", + "status": "missing", + "note": "Chameleon is released by request only; DRCT-2M is not in SynthScope yet." + }, + { + "benchmark": "Mirage-Test (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=96.48 / acc=78.46 / AUROC=96.65 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=96.48 / acc=78.46 / AUROC=96.65", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 78.46 in the paper's metric (balanced acc 80.37, mAP 96.48) (tag omniaid_mirage_test, job 265937)." + }, + { + "benchmark": "SynthWildX (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=95.52 / acc=86.02 / AUROC=95.31 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=95.52 / acc=86.02 / AUROC=95.31", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 86.02 (mAP 95.52, AUROC 95.31) (tag omniaid_synthwildx, job 265858)." + }, + { + "benchmark": "WildRF (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=97.28 / acc=88.36 / AUROC=96.91 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=97.28 / acc=88.36 / AUROC=96.91", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 88.36 (mAP 97.28, AUROC 96.91) (tag omniaid_wildrf, job 265858)." + }, + { + "benchmark": "BFree-Online (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=91.7 / acc=70.26 / AUROC=87.18 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=91.7 / acc=70.26 / AUROC=87.18", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 70.26 (mAP 91.70, AUROC 87.18) (tag omniaid_bfree_online, job 265858)." + }, + { + "benchmark": "RealChain (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=84.02 / acc=61.71 / AUROC=83.63 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=84.02 / acc=61.71 / AUROC=83.63", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 61.71 (mAP 84.02, AUROC 83.63) (tag omniaid_realchain, job 265858)." + }, + { + "benchmark": "EvalGEN (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=96.46 / acc=74.14 / AUROC=93.08 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=96.46 / acc=74.14 / AUROC=93.08", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 74.14 (mAP 96.46, AUROC 93.08) (tag omniaid_evalgen, job 265858)." + }, + { + "benchmark": "RRDataset (v2 release, served)", + "metric": "balanced accuracy", + "paper": "\u2014", + "ours": "mAP=84.19 / acc=71.75 / AUROC=82.32 (official wts)", + "ours_synthscope": "", + "ours_official": "mAP=84.19 / acc=71.75 / AUROC=82.32", + "status": "reference", + "note": "omniaid (v2) through SynthScope: 71.75 (mAP 84.19, AUROC 82.32) (tag omniaid_rrdataset, job 265858)." + } + ], "pgc": [ { "benchmark": "GenImage", @@ -604,21 +746,21 @@ "benchmark": "UniversalFakeDetect (UFD)", "metric": "AP", "paper": "map=98.0", - "ours": "SynthScope mAP=99.01 \u00b7 official mAP=99.34 / acc=94.16", - "ours_synthscope": "mAP=99.01", + "ours": "SynthScope mAP=98.99 / acc=91.96 / AUROC=98.92 \u00b7 official mAP=99.34 / acc=94.16", + "ours_synthscope": "mAP=98.99 / acc=91.96 / AUROC=98.92", "ours_official": "mAP=99.34 / acc=94.16", "status": "reproduced", - "note": "Reproduced (official-wts 99.34, SynthScope retrain 99.01) >= paper 98.0; no gap." + "note": "Reproduced (official weights 99.34; the official recipe trained in SynthScope, step 500: 98.99) >= paper 98.0." }, { "benchmark": "ForenSynths (CNNDetection test)", "metric": "AP", "paper": "\u2014", - "ours": "SynthScope mAP=89.03 \u00b7 official mAP=96.25 / acc=83.73", - "ours_synthscope": "mAP=89.03", + "ours": "SynthScope mAP=97.41 / acc=85.89 / AUROC=97.63 \u00b7 official mAP=96.25 / acc=83.73", + "ours_synthscope": "mAP=97.41 / acc=85.89 / AUROC=97.63", "ours_official": "mAP=96.25 / acc=83.73", "status": "reference", - "note": "Official-weights-in-SynthScope reproduces (96.25 mAP). SynthScope retrain (89.03) below on the GAN generators (ProGAN-trained LoRA-DINOv2 transfers less to cyclegan/gaugan); residual cross-generator gap, not a code defect." + "note": "Not a paper number. The official recipe trained in SynthScope (step 500) gives FS-8 97.41 mAP, above the official checkpoint (96.41 on the same evaluation); the earlier 89.03 retrain overshot the early transfer peak." }, { "benchmark": "AIGIBench (AIGI)", diff --git a/models/image/d3qe/README.md b/models/image/d3qe/README.md index 67811c31fc5ca5b155f91b70be111415a2185359..b363b47de36d09e8f9d120aac9ed91a660c17e0e 100644 --- a/models/image/d3qe/README.md +++ b/models/image/d3qe/README.md @@ -32,7 +32,7 @@ is downloaded to `~/.cache/clip` (or `$SYNTHSCOPE_CLIP_DIR`) and the LlamaGen VQ | `official/` = the official `networks/D3QE.py`, `vq_model.py` and `clip/model.py`, **byte-identical bodies** | `SynthScopeD3QEModel` subclasses the official `D3QE`, so the checkpoint loads key-for-key; MIT attribution in the headers. | | `official/clip/clip.py` reduced to the official `load()` path D3QE uses (CPU, non-JIT) | The text tokenizer is not needed; `from pkg_resources import packaging`, which current setuptools no longer provides, became `import packaging.version`. | | `pretrained_prefixes = ("clip_model.", "vq_model.")` | The checkpoint does not contain the frozen CLIP and VQ weights (the official `eval.py` falls back to `strict=False`). The engine accepts missing keys only under these prefixes; every trained tensor still loads strictly. | -| `compute_loss` = BCE on the logit, labels passed to `forward` | The official objective; the forward pass updates the codebook statistics during training as in the official code. ARForensics training is not reproduced here. | +| `compute_loss` = BCE on the logit, labels passed to `forward` | The official objective; the forward pass updates the codebook statistics during training as in the official code. The full official recipe is `experiments/image/d3qe.sh` (see *Training in SynthScope* below). | Two properties of the released code are kept on purpose, because they decide the released scores: * the VQ tokenizer sees the [0, 1] image: `forward` builds a [-1, 1] copy but encodes the unnormalized one; @@ -55,9 +55,9 @@ accuracy equals SynthScope's balanced accuracy. | Benchmark | Paper (acc / AP) | SynthScope_official (acc / AP) | SynthScope_train | Status | |---|---|---|---|---| -| **ARForensics** (7 AR generators, all 84,000 test images) | 82.11 / 92.07 | **81.74 / 92.22** (AUROC 91.91) | — | **reproduced** | -| **ForenSynths** (the paper's 7 GANs) | 83.73 / 92.23 | **83.45 / 92.23** (AUROC 90.65) | — | **reproduced** | -| **GenImage** (the paper's 6 subsets) | 78.61 / 89.60 | **79.44 / 90.94** (AUROC 90.49) | — | **reproduced** on the mean; per-subset gaps up to 6.8 pt | +| **ARForensics** (7 AR generators, all 84,000 test images) | 82.11 / 92.07 | **81.74 / 92.22** (AUROC 91.91) | 79.83 / 91.73 | official **reproduced**; trained: AP matches, acc 2.3 below | +| **ForenSynths** (the paper's 7 GANs) | 83.73 / 92.23 | **83.45 / 92.23** (AUROC 90.65) | **82.14 / 91.35** | **reproduced** (official and trained) | +| **GenImage** (the paper's 6 subsets) | 78.61 / 89.60 | **79.44 / 90.94** (AUROC 90.49) | **77.01 / 90.11** | **reproduced** on the mean (official and trained); per-subset gaps up to 6.8 pt | Per generator (SynthScope vs paper, accuracy): @@ -78,6 +78,23 @@ BFree-Online 47.03, RealChain 49.49, EvalGEN 62.21 (balanced; AUROC 83.94), RRDa Runs: jobs 264795 (ARForensics, ForenSynths, GenImage) and 264796 (in-the-wild sets); results in `repro/bench_results/d3qe_*.json`. +## Training in SynthScope + +`experiments/image/d3qe.sh` runs the official recipe (`train.py` and its `TrainOptions` defaults) through SynthScope's +trainer: ARForensics train (100k LlamaGen vs 100k ImageNet) and val; short side 256, then the CNNDetection blur/JPEG +augmentation (p 0.1 each; `CNNDetAugment`, identical to the official `data_augment` over 300 seeded draws), RandomCrop +256, flip, no normalization; AdamW lr 8e-5, weight decay 0.01, batch 32 on one GPU, 10 epochs; the best-validation +checkpoint is kept and the learning rate is divided by 10 after 5 epochs without improvement (trainer `plateau`). It runs +as a plain single process, not through the repo's DeepSpeed accelerate config, whose gradient clipping the official +recipe does not have. + +Our run (job 265064, 15 h on one A5000) improved its validation accuracy every epoch (88.5 → 94.8), so, as in the +released checkpoint (62,500 steps), the last epoch is the best and the learning rate never dropped. Against the paper it +gives 79.83 / 91.73 (ARForensics), 82.14 / 91.35 (ForenSynths-7) and 77.01 / 90.11 (GenImage-6): AP reproduces everywhere +(within 0.9 pt), accuracy is 1.6 to 2.3 pt lower. Per generator the trained model matches the official one closely +(LlamaGen 97.49 vs 97.16) but calls a few more unseen-generator fakes real (real accuracy 97.4 vs 96.4), which moves the +fixed-threshold accuracy, not the ranking. Results: `repro/bench_results/d3qe_trained_*.json` (job 265826). + ## Usage ```bash diff --git a/models/image/d3qe/d3qe.py b/models/image/d3qe/d3qe.py index 4476f9eac73f7800d98486b3d6ad6d78536923c0..c19b5ea2bb72b4c1b843936d8b866cb9944612da 100644 --- a/models/image/d3qe/d3qe.py +++ b/models/image/d3qe/d3qe.py @@ -26,6 +26,7 @@ Input contract: ``forward`` takes the tensor of the official test transform — import os from pathlib import Path +import torch import torch.nn as nn from omegaconf import DictConfig @@ -58,6 +59,7 @@ class SynthScopeD3QEModel(D3QE): # Official objective (networks/trainer.py): BCE on the single logit; passing the labels lets the forward pass # update the codebook-frequency statistics during training, as in the official code. images, labels, _folder_ids = batch - outputs = self(images, labels) - loss = self.criterion(outputs.squeeze(1), labels.float()) - return {"loss": loss, "outputs": outputs} + logits = self(images, labels).squeeze(1) + loss = self.criterion(logits, labels.float()) + # Two-column [-logit, logit] so the trainer's metrics (argmax / softmax[:, 1]) read the single logit right. + return {"loss": loss, "outputs": torch.stack([-logits, logits], dim=1)} diff --git a/models/image/dda/dda.py b/models/image/dda/dda.py index 8d36bead3440265a73aecbcddbde13bd95640c50..7f9b2fc9abc58231209229ad3b4aca2bbcb87b88 100644 --- a/models/image/dda/dda.py +++ b/models/image/dda/dda.py @@ -62,6 +62,7 @@ class SynthScopeDDAModel(nn.Module): def compute_loss(self, batch): images, labels, _folder_ids = batch - outputs = self.forward(images) - loss = self.criterion(outputs.squeeze(1), labels.float()) - return {"loss": loss, "outputs": outputs} + logits = self.forward(images).squeeze(1) + loss = self.criterion(logits, labels.float()) + # Two-column [-logit, logit] so the trainer's metrics (argmax / softmax[:, 1]) read the single logit right. + return {"loss": loss, "outputs": torch.stack([-logits, logits], dim=1)} diff --git a/models/image/npr/README.md b/models/image/npr/README.md index bad72a1d7a81a97f05b48458ca0e465dc4a77ba2..937b3c890fd1fd65ebe239c3b0efbfd557d6cb27 100644 --- a/models/image/npr/README.md +++ b/models/image/npr/README.md @@ -69,18 +69,43 @@ The paper reports both accuracy and average precision, with the main result show All benchmarks the paper reports, with our numbers (verified, traced to `repro/bench_results/` + `repro/benchmark_specs/`). -Values are `mAP / acc`. **SynthScope_official** = the authors' released weights run through the SynthScope class / eval. **SynthScope_train** = the SynthScope class trained from scratch by us. For all NPR entries we did not train from scratch, so SynthScope_train is `—`. +Values are `mAP / acc`. **SynthScope_official** = the authors' released weights run through the SynthScope class / eval. **SynthScope_train** = the SynthScope class trained from scratch by us with the official recipe (`experiments/image/npr.sh`; see *Training in SynthScope* below), evaluated so far on Tables 1 and 4. | Benchmark | Paper | SynthScope_official | SynthScope_train | Status | |---|---|---|---|---| -| Table 1 — ForenSynths / Cross-GAN (8) | 96.1 / 92.5 | 94.22 / 92.3 | — | official | +| Table 1 — ForenSynths / Cross-GAN (8) | 96.1 / 92.5 | 94.22 / 92.3 | 93.83 / 84.09 | official; trained: AP close, acc below (as the official training code here) | | Table 2 — GANGen-Detection (9) | 96.6 / 93.2 | 96.49 / 94.81 | — | official | | Table 3 — DiffusionForensics / DIRE | 99.8 / 95.3 | 99.4 / 95.03 | — | official | -| Table 4 — UniversalFakeDetect / Ojha | acc 95.2 | 96.69 / 93.94 | — | official | +| Table 4 — UniversalFakeDetect / Ojha | acc 95.2 | 96.69 / 93.94 | 97.49 / 88.91 | official; trained: AP matches, acc below | | Table 5 — Diffusion1kStep | acc 80.1 | 83.47 / 79.13 | — | official | **reproduced** = SynthScope-trained checkpoint matches the paper · **official** = official weights loaded into the SynthScope class match (the SynthScope-trained number is shown alongside where we trained from scratch) · **below** = gap remains · **reference** = paper reports no comparable number · **missing** = not yet evaluated. +## Training in SynthScope + +`experiments/image/npr.sh` runs the official recipe (the README's `train.py` command and `networks/trainer.py`): ProGAN +4-class (car, cat, chair, horse; 144,024 images), `Resize((256, 256))` → RandomCrop 224 → flip → ImageNet normalization +(no blur/JPEG: the official `data_augment` call is commented out), Adam lr 2e-4 without weight decay, batch 32 on one +GPU, 50 epochs, learning rate ×0.9 after epochs 10, 20, 30 and 40, last-epoch checkpoint, seed 100. The SynthScope NPR +uses the official convolution initialization for this. + +To tell our pipeline apart from run-to-run variance, we also ran the **official `train.py`, unmodified**, on the same +data and machine, and scored every checkpoint with the same `bench_eval` command (`mAP / acc`): + +| Checkpoint | Table 1 — ForenSynths (8 GANs) | Table 4 — UniversalFakeDetect | +|---|---|---| +| official `NPR.pth` (served as `npr.pth`) | 94.22 / 92.30 | 97.16 / 94.57 | +| official `model_epoch_last_3090.pth` | 92.81 / 91.83 | 96.69 / 93.94 | +| official `train.py` rerun here | 92.72 / 81.47 | 96.56 / 87.61 | +| **SynthScope recipe (ours)** | **93.83 / 84.09** | **97.49 / 88.91** | + +Our recipe matches the official code's own rerun on every number (it is slightly better), so the SynthScope pipeline is +faithful; neither run reaches the released checkpoints' accuracy. AP — how well real and fake are ranked — reproduces; +the accuracy gap is a threshold shift (our models call more unseen-generator fakes real, e.g. CycleGAN fake recall 48 vs +96 for `NPR.pth`), which the released checkpoints did not show and which this environment (PyTorch 2.11 instead of 1.8, +another GPU) did not reproduce. Runs: training job 265720, official-code rerun 265815, evaluation 265808/265814/266028 +(`repro/bench_results/npr_{official_nprpth,official3090,officialcode_rerun,trained_last,trained_best}_*.json`). + ## Model details The SynthScope implementation lives in: diff --git a/models/image/npr/npr.py b/models/image/npr/npr.py index 1ef8b7b299205b8ed341f8d680ea1d2b360e0e36..1b082a21a1933a65a4e2b72c797fc750940a7b80 100644 --- a/models/image/npr/npr.py +++ b/models/image/npr/npr.py @@ -39,6 +39,13 @@ class NPR(nn.Module): ) self.head = nn.Linear(layer2_ch, 1) self.criterion = torch.nn.BCEWithLogitsLoss() + # Official initialization (networks/resnet.py ResNet.__init__); only matters when training from scratch. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) def NPR_transform(self, x: torch.Tensor) -> torch.Tensor: diff --git a/models/image/omniaid/README.md b/models/image/omniaid/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5fda7637b89b103604891a45e326a43bc3a57443 --- /dev/null +++ b/models/image/omniaid/README.md @@ -0,0 +1,88 @@ +# OmniAID (`omniaid`, `omniaid_v1`, `omniaid_genimage`) + +**OmniAID: Decoupling Semantics and Artifacts for Universal AI-Generated Image Detection in the Wild** — +[paper (arXiv 2511.08423)](https://arxiv.org/abs/2511.08423) · [official code](https://github.com/yunncheng/OmniAID) · +[Mirage-Test](https://huggingface.co/datasets/Yunncheng/Mirage-Test) + +A detector trained on one kind of content (say, landscapes) often fails on another (faces), because it mixes up *what* +is shown with *how* it was made. OmniAID splits these apart with a hybrid mixture of experts on CLIP ViT-L/14@336: +semantic experts (human, animal, object, scene, ...) are routed per image, and an artifact expert is always on. The +authors also built **Mirage**, a train/test set of images from recent generators tuned for realism. + +## Pipeline (official, reproduced exactly) + +``` +image ─ Resize((336, 336)) [v2] | Resize((512, 512)) → Resize((336, 336)) [v1-era releases] + ─ ToTensor → CLIP normalization + ├ frozen CLIP vision tower → pooler output → router → top-k semantic experts + └ MoE ViT-L/14: every attention projection = shared main weights (SVD) + the routed semantic experts + + the always-on artifact expert + → ln_post(CLS) → Linear(1024, 2) → logits; P(fake) = softmax(logits)[:, 1] (the official `prob`) +``` + +Three official releases (Hugging Face `Yunncheng/OmniAID`, MIT), each a full model loaded with `strict=True` as in the +official eval: + +| key | release | trained on | experts × rank, router, top-k | preprocessing | +|---|---|---|---|---| +| `omniaid` | `checkpoint_omniaid_v2.pth` (recommended; served on the demo) | Mirage-Train | 6 × 1, 256, top-2 | single resize to 336 | +| `omniaid_v1` | `checkpoint_omniaid_v1.pth` (the paper's OmniAID-Mirage) | Mirage-Train | 6 × 8, 256, top-2 | 512 → 336 | +| `omniaid_genimage` | `checkpoint_omniaid_genimage_sd14.pth` (the paper's academic tables) | GenImage SD v1.4 | 3 × 4, 64, top-1 | 512 → 336 | + +The expert counts, ranks and router widths were read from each checkpoint's tensor shapes (the README's +`config_omniaid_v1/v2.json` are not in the repo); top-k comes from the official configs and `reward/clean_test.py`. +The v1-era double resize is the pipeline the official code says to use for v1 weights; for the GenImage release it is +confirmed by the results below (it reproduces the paper). + +## SynthScope adaptations + +| choice | reason | +|---|---| +| `official_arch.py` = the official `models/OmniAID.py`, **byte-identical body** | `SynthScopeOmniAIDModel` subclasses the official `OmniAID`, so the checkpoints load key-for-key; MIT attribution in the header. | +| `forward` returns the 2-class logits (`cls`) | SynthScope's `ce` convention gives softmax[:, 1], the official `prob`. | +| one registry key per release | Each release has its own expert configuration and preprocessing. | +| `compute_loss` = cross-entropy on the logits | **The official recipe trains in two stages** (each semantic expert with hard routing, then the router with the artifact expert fixed) with orthogonality, load-balancing and gating losses; that recipe is not reproduced here. | + +## Fidelity check (identical to the official model) + +The official `OmniAID` (built from the release's config and loaded `strict=True`, as `main_finetune.py` does) and +SynthScope's class, for all three releases on an A5000: **every state-dict tensor identical** (2804 / 2804 / 1940, max +|diff| 0.0), the registry transforms equal the official v2 / v1 pipelines pixel for pixel, and **max |logit diff| 0.0 +with 0 verdict flips** on 24 GenImage images (ADM, BigGAN, Midjourney, Wukong; real and fake). Job 265851. + +## Results + +Paper metric: accuracy at 0.5 per subset, mean over subsets (Tables 2, 3, 6). Every subset used here is balanced except +Mirage-Test's Anime (fakes only), so accuracy equals SynthScope's balanced accuracy. + +| Benchmark | Release | Paper | SynthScope_official | SynthScope_train | Status | +|---|---|---|---|---|---| +| **GenImage** (8 subsets) | `omniaid_genimage` | 95.94 | **95.82** (AP 99.02) | — | **reproduced** (every subset within 1.5 pt) | +| **AIGCDetectBenchmark** (17) | `omniaid_genimage` | 88.87 | **88.87** (AP 97.88) | — | **reproduced** exactly | +| **Mirage-Test** (5 categories) | `omniaid_genimage` | 51.10 | **51.10** | — | **reproduced** exactly (every category) | +| **GenImage** (8 subsets) | `omniaid_v1` | 97.24 | **97.16** (AP 99.20) | — | **reproduced** (every subset within 0.6 pt) | +| **AIGCDetectBenchmark** (17) | `omniaid_v1` | 92.88 | **92.88** (AP 98.93) | — | **reproduced** exactly | +| **Mirage-Test** (5 categories) | `omniaid_v1` | 88.39 | **88.39** | — | **reproduced** exactly | +| Chameleon / DRCT-2M | `omniaid_genimage` | 77.35 / 88.21 | — | — | missing (Chameleon is by request; DRCT-2M is not in SynthScope) | + +GenImage per subset (SynthScope vs paper): `omniaid_genimage` Midjourney 87.20 vs 85.7 · SD v1.4 98.92 vs 98.9 · SD v1.5 +98.77 vs 98.8 · ADM 90.40 vs 91.4 · GLIDE 98.97 vs 98.7 · Wukong 97.50 vs 98.1 · VQDM 96.07 vs 97.3 · BigGAN 98.70 vs +98.7; `omniaid_v1` Midjourney 98.20 vs 98.0 · SD v1.4 98.69 vs 98.7 · SD v1.5 98.63 vs 98.4 · ADM 88.90 vs 89.5 · GLIDE +98.20 vs 98.3 · Wukong 98.43 vs 98.6 · VQDM 98.40 vs 98.4 · BigGAN 97.87 vs 98.1. + +**Mirage-Test protocol.** The paper's mean averages the balanced accuracy ("Overall") of Human, Animal, Object and Scene with the fake accuracy of Anime, which has no reals; the loader pairs Anime with the other categories' reals, and SynthScope reports the same mean. `omniaid_genimage`: Human 59.38, Animal 55.90, Object 49.02, Scene 46.52, Anime 44.67 (paper Table 3: 59.38, 55.90, 49.02, 46.53, 44.67); `omniaid_v1`: Human 93.69, Animal 81.37, Object 84.92, Scene 87.07, Anime 94.92 (paper: 93.69, 81.37, 84.92, 87.07, 94.92). The recommended v2 release (`omniaid`, not in the paper) gives 78.46 (AP 96.48): Human 88.82, Animal 77.20, Object 75.09, Scene 70.89, Anime 80.28. + +**In the wild (`omniaid`, v2; new measurements, not in the paper)** — balanced accuracy / AP: SynthWildX 86.02 / 95.52, +WildRF 88.36 / 97.28, BFree-Online 70.26 / 91.70, RealChain 61.71 / 84.02, EvalGEN 74.14 / 96.46, RRDataset 71.75 / +84.19. See [docs/WILD_BENCHMARKS.md](../../../docs/WILD_BENCHMARKS.md). + +Runs: jobs 265856 (`omniaid_genimage`), 265857 (`omniaid_v1`), 265858 (`omniaid`), 265935-265937 (Mirage-Test); results in +`repro/bench_results/omniaid*.json`. + +## Usage + +```bash +synthscope detect photo.png --model omniaid # v2; needs omniaid.pth in $SYNTHSCOPE_CKPT_DIR + HF access for CLIP +python benchmark/bench_eval.py --model omniaid_genimage --dataset genimage_full --root /data/GenImage_full +python benchmark/bench_eval.py --model omniaid_v1 --dataset mirage_test --root /data/MirageTest +``` diff --git a/models/image/omniaid/__init__.py b/models/image/omniaid/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/image/omniaid/official_arch.py b/models/image/omniaid/official_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..0fdff803ca1e19a75f972c578fcf51664c20cc29 --- /dev/null +++ b/models/image/omniaid/official_arch.py @@ -0,0 +1,681 @@ +# Vendored from github.com/yunncheng/OmniAID (MIT License per its README and Hugging Face model card), file models/OmniAID.py. +# Unchanged below this header, so the released checkpoints load key-for-key (strict=True, as the official eval does). +import torch +import torch.nn as nn +import torch.nn.functional as F +from typing import Optional, Tuple, Dict +from transformers import CLIPModel, CLIPVisionConfig +from transformers.models.clip.modeling_clip import CLIPVisionEmbeddings, CLIPMLP +from torch.utils.checkpoint import checkpoint + +class OmniAID(nn.Module): + def __init__(self, config=None): + super(OmniAID, self).__init__() + self.config = config + self.num_experts = config.num_experts + self.rank_per_expert = config.rank_per_expert + + self.moe_lambda_orth = config.moe_lambda_orth + self.moe_lambda_balance = config.moe_lambda_balance + self.moe_lambda_gating_cls = config.moe_lambda_gating_cls + self.svd_dropout = config.dropout + + self.moe_router_hidden_dim = config.moe_router_hidden_dim + self.top_k = config.moe_top_k + + self.gradient_checkpointing = config.gradient_checkpointing_enable + + self.gating_criterion = torch.nn.CrossEntropyLoss() + + self.training_mode = "standard" # Default runtime mode, can be changed by set_training_mode + self.active_expert_idx = None + + # This property is fixed after initialization + self.is_hybrid = config.is_hybrid + + if self.is_hybrid: + print("Initializing in HYBRID MoE architectural mode.") + # Use the last expert as the artifact expert + self.artifact_expert_idx = config.num_experts - 1 + self.num_semantic_experts = config.num_experts - 1 + if self.num_semantic_experts <= 0: + raise ValueError("num_experts must be at least 2 for the Hybrid MoE model (1 fixed, >=1 semantic).") + gating_num_experts = self.num_semantic_experts + else: + print("Initializing in STANDARD MoE architectural mode.") + self.artifact_expert_idx = -1 # A value that will never match an expert index + gating_num_experts = self.num_experts + + pretrained_path = config.CLIP_path + + # Initialize an independent Vision Model as the feature extractor. This model will be used exclusively to generate features for the router. + self.feature_extractor = CLIPModel.from_pretrained(pretrained_path).vision_model + self.feature_extractor.eval() + + vision_config = self.feature_extractor.config + self.hidden_size = vision_config.hidden_size + + # Calculate rank allocation + total_rank = vision_config.hidden_size + residual_rank = self.rank_per_expert + + if residual_rank >= total_rank: + raise ValueError( + f"The total rank for experts ({residual_rank}) must be less than the total rank ({total_rank}). " + f"Please reduce num_experts ({self.num_experts}) or rank_per_expert ({self.rank_per_expert})." + ) + + r_main = total_rank - residual_rank + print(f"Rank allocation: total_rank={total_rank}, r_main={r_main}, num_experts={self.num_experts}, rank_per_expert={self.rank_per_expert}") + + # Build the MoE backbone network + self.embeddings = CLIPVisionEmbeddings(vision_config) + self.ln_pre = nn.LayerNorm(vision_config.hidden_size) + self.encoder_layers = nn.ModuleList([ + ViTMoELayer(vision_config, self.num_experts, r_main, self.rank_per_expert, self.artifact_expert_idx, self.svd_dropout) + for _ in range(vision_config.num_hidden_layers) + ]) + self.ln_post = nn.LayerNorm(self.hidden_size, eps=vision_config.layer_norm_eps) + self.gating_network = GatingNetwork( + input_dim=self.hidden_size, + num_experts=gating_num_experts, + hidden_dim=self.moe_router_hidden_dim, + top_k=self.top_k + ) + self.head = nn.Linear(self.hidden_size, 2) + + self.load_and_replace_from_pretrained(self.feature_extractor) + + def load_and_replace_from_pretrained(self, pretrained_model): + self.embeddings.load_state_dict(pretrained_model.embeddings.state_dict()) + self.ln_pre.load_state_dict(pretrained_model.pre_layrnorm.state_dict()) + self.ln_post.load_state_dict(pretrained_model.post_layernorm.state_dict()) + + for moe_layer, pretrained_layer in zip(self.encoder_layers, pretrained_model.encoder.layers): + for proj_name in ['q_proj', 'k_proj', 'v_proj', 'out_proj']: + self._replace_linear_with_svd_moe( + getattr(pretrained_layer.self_attn, proj_name), + getattr(moe_layer.self_attn, proj_name) + ) + moe_layer.layer_norm1.load_state_dict(pretrained_layer.layer_norm1.state_dict()) + moe_layer.mlp.load_state_dict(pretrained_layer.mlp.state_dict()) + moe_layer.layer_norm2.load_state_dict(pretrained_layer.layer_norm2.state_dict()) + + # Freeze non-MoE parameters + for name, param in self.named_parameters(): + if 'experts' not in name and 'gating' not in name and 'head' not in name: + param.requires_grad = False + + def _replace_linear_with_svd_moe(self, original_module: nn.Linear, moe_module): + original_weight = original_module.weight.data + moe_module.weight_original_fnorm.data.copy_(torch.norm(original_weight, p='fro')) + if original_module.bias is not None: + moe_module.bias.data.copy_(original_module.bias.data) + + U, S, Vh = torch.linalg.svd(original_weight, full_matrices=False) + + # Initialize the main (shared) weight matrix + r = moe_module.r_main + U_r, S_r, Vh_r = U[:, :r], S[:r], Vh[:r, :] + moe_module.weight_main.data.copy_(U_r @ torch.diag(S_r) @ Vh_r) + + moe_module.U_r.data.copy_(U_r) + moe_module.V_r.data.copy_(Vh_r) + + # Define the SINGLE, SHARED chunk from ALL remaining singular values + chunk_start_rank = r + + # Check if there are any singular values left + if chunk_start_rank >= len(S): + # If not, initialize all experts to zero + for i in range(moe_module.num_experts): + moe_module.U_experts[i].data.zero_() + moe_module.S_experts[i].data.zero_() + moe_module.V_experts[i].data.zero_() + return + + U_chunk, S_chunk, Vh_chunk = U[:, chunk_start_rank:], S[chunk_start_rank:], Vh[chunk_start_rank:, :] + + # The expert's parameter dimension is fixed at rank_per_expert. We must pad the chunk if its actual rank is smaller + actual_chunk_rank = U_chunk.shape[1] + + if actual_chunk_rank < moe_module.rank_per_expert: + pad_rank = moe_module.rank_per_expert - actual_chunk_rank + U_chunk = F.pad(U_chunk, (0, pad_rank)) + S_chunk = F.pad(S_chunk, (0, pad_rank)) + Vh_chunk = F.pad(Vh_chunk, (0, 0, 0, pad_rank)) + + # Assign the SAME shared chunk to all experts + for i in range(moe_module.num_experts): + moe_module.U_experts[i].data.copy_(U_chunk) + moe_module.S_experts[i].data.copy_(S_chunk) + moe_module.V_experts[i].data.copy_(Vh_chunk) + + def set_training_mode(self, mode: str, active_expert_idx: int = None, trainable_expert_indices: list = None): + """ + Sets the training mode of the model, allowing specific parts of the parameters to be frozen/unfrozen. + + Args: + mode (str): Training mode. Possible values are: + - 'hard_sampling': Stage 1. Unfreezes one active expert and the head. + - 'router_training': Stage 2. Unfreezes the router and the head. Optionally unfreezes specified experts. + - 'standard': End-to-end. Unfreezes the router, head, and all experts. + active_expert_idx (int, optional): The index of the expert to be trained in 'hard_sampling' mode. Required for this mode. + trainable_expert_indices (list, optional): A list of expert indices to also unfreeze during 'router_training' mode. + Allows for joint training of the router and specific experts. + """ + if mode not in ['hard_sampling', 'router_training', 'standard']: + raise ValueError("Training mode must be one of 'hard_sampling', 'router_training', or 'standard'") + + if mode == 'hard_sampling' and (active_expert_idx is None or not isinstance(active_expert_idx, int) or active_expert_idx < 0): + raise ValueError("A valid integer active_expert_idx must be provided for 'hard_sampling' mode.") + + if mode == 'router_training' and trainable_expert_indices is not None and not isinstance(trainable_expert_indices, list): + raise ValueError("'trainable_expert_indices' must be a list of integers.") + + + # Store the state in the model instance + self.training_mode = mode + self.active_expert_idx = active_expert_idx if mode == 'hard_sampling' else None + + for param in self.parameters(): + param.requires_grad = False + + if mode == 'standard': + # In standard MoE training, train the router, all experts, and the final head + for param in self.gating_network.parameters(): + param.requires_grad = True + + for param in self.head.parameters(): + param.requires_grad = True + + for layer in self.encoder_layers: + for proj_name in ['q_proj', 'k_proj', 'v_proj', 'out_proj']: + moe_linear_layer = getattr(layer.self_attn, proj_name) + for param in moe_linear_layer.U_experts: + param.requires_grad = True + for param in moe_linear_layer.S_experts: + param.requires_grad = True + for param in moe_linear_layer.V_experts: + param.requires_grad = True + + elif mode == 'router_training': + print("Unfreezing: Gating Network and Head.") + for param in self.gating_network.parameters(): + param.requires_grad = True + for param in self.head.parameters(): + param.requires_grad = True + + # Optionally unfreeze specific experts during router training + if trainable_expert_indices: + print(f"Additionally unfreezing experts with indices: {trainable_expert_indices}") + for expert_idx in trainable_expert_indices: + if not 0 <= expert_idx < self.num_experts: + raise ValueError(f"trainable_expert_index ({expert_idx}) is out of bounds for num_experts ({self.num_experts}).") + + for layer in self.encoder_layers: + for proj_name in ['q_proj', 'k_proj', 'v_proj', 'out_proj']: + moe_linear_layer = getattr(layer.self_attn, proj_name) + moe_linear_layer.U_experts[expert_idx].requires_grad = True + moe_linear_layer.S_experts[expert_idx].requires_grad = True + moe_linear_layer.V_experts[expert_idx].requires_grad = True + + elif mode == 'hard_sampling': + # In hard_sampling stage, unfreeze only the parameters of the single active expert + if self.active_expert_idx >= self.num_experts: + raise ValueError(f"active_expert_idx ({self.active_expert_idx}) is out of bounds for num_experts ({self.num_experts}).") + + for layer in self.encoder_layers: + for proj_name in ['q_proj', 'k_proj', 'v_proj', 'out_proj']: + moe_linear_layer = getattr(layer.self_attn, proj_name) + + # Unfreeze the parameters (U, S, V) for the specified active expert by direct indexing + moe_linear_layer.U_experts[self.active_expert_idx].requires_grad = True + moe_linear_layer.S_experts[self.active_expert_idx].requires_grad = True + moe_linear_layer.V_experts[self.active_expert_idx].requires_grad = True + + for param in self.head.parameters(): + param.requires_grad = True + + print(f"Successfully set training mode to '{self.training_mode}'" + (f" (active expert: {self.active_expert_idx})" if self.training_mode == 'hard_sampling' else "")) + + # print("Currently trainable parameters:") + # for name, param in self.named_parameters(): + # if param.requires_grad: + # print(f"- {name}") + + # For checkpoint + @staticmethod + def run_moe_layer(module, hidden_states, gating_outputs): + return module(hidden_states, gating_outputs=gating_outputs)[0] + + def forward(self, images) -> dict: + batch_size = images.size(0) + + with torch.no_grad(): + # Use the pooler_output as the routing feature + routing_features = self.feature_extractor(images, output_hidden_states=False).pooler_output + + hidden_states = self.embeddings(images) + hidden_states = self.ln_pre(hidden_states) + + gating_outputs = {} + + # Stage 1 expert training ('hard_sampling') + if self.training_mode == 'hard_sampling': + if self.active_expert_idx is None: + raise ValueError("In hard_sampling mode, active_expert_idx must be set.") + + hard_sampling_target = torch.full((batch_size,), self.active_expert_idx, dtype=torch.long, device=hidden_states.device) + gating_outputs['top_k_indices'] = hard_sampling_target.unsqueeze(1) + gating_outputs['top_k_gates'] = torch.ones_like(gating_outputs['top_k_indices'], dtype=hidden_states.dtype) + gating_outputs['balance_loss'] = torch.tensor(0.0, device=hidden_states.device) + gating_outputs['gating_logits'] = None + + # Stage 2 routing ('router_training') or inference for a Hybrid model + elif self.is_hybrid: + # Create a "pass-through" gating output for the fixed artifact expert (the last expert) + artifact_expert_indices = torch.full( + (batch_size, 1), + self.artifact_expert_idx, # Use the last expert index + dtype=torch.long, + device=hidden_states.device + ) + artifact_expert_gates = torch.ones( + (batch_size, 1), + dtype=hidden_states.dtype, + device=hidden_states.device + ) + + # Get routing decisions for the SEMANTIC experts (experts 0, 1, ..., N-2) + semantic_gating_outputs = self.gating_network(routing_features) + + semantic_expert_indices = semantic_gating_outputs['top_k_indices'] + + # Combine the routed semantic experts and the fixed artifact expert + gating_outputs['top_k_indices'] = torch.cat([semantic_expert_indices, artifact_expert_indices], dim=1) + gating_outputs['top_k_gates'] = torch.cat([semantic_gating_outputs['top_k_gates'], artifact_expert_gates], dim=1) + + gating_outputs['balance_loss'] = semantic_gating_outputs['balance_loss'] + gating_outputs['gating_logits'] = semantic_gating_outputs['gating_logits'] + + # Standard end-to-end training or inference for a Standard MoE model + else: + gating_outputs = self.gating_network(routing_features) + + final_gates = torch.zeros(batch_size, self.num_experts, device=hidden_states.device, dtype=gating_outputs['top_k_gates'].dtype) + + final_gates.scatter_(-1, gating_outputs['top_k_indices'], gating_outputs['top_k_gates']) + + for layer_module in self.encoder_layers: + if self.gradient_checkpointing and self.training: + hidden_states = checkpoint( + self.run_moe_layer, + layer_module, + hidden_states, + gating_outputs, + use_reentrant=False + ) + else: + hidden_states = layer_module( + hidden_states, + gating_outputs=gating_outputs, + )[0] + + pooled_output = self.ln_post(hidden_states[:, 0, :]) + pred = self.head(pooled_output) + prob = torch.softmax(pred, dim=1)[:, 1] + + return { + 'cls': pred, + 'prob': prob, + 'balance_loss': gating_outputs['balance_loss'], + 'gating_logits': gating_outputs['gating_logits'], + 'final_gates': final_gates + } + + def get_losses(self, pred_dict: dict, labels, expert_domain_labels, criterion) -> dict: + pred = pred_dict['cls'] + classification_loss = criterion(pred, labels) + + orth_loss, load_balancing_loss = torch.tensor(0.0, device=pred.device), torch.tensor(0.0, device=pred.device) + gating_classification_loss = torch.tensor(0.0, device=pred.device) + + num_moe_layers = sum(1 for module in self.modules() if isinstance(module, SVDMoeLinear)) + + # Orthogonal loss is not calculated during router_training + if self.training_mode != 'router_training' and num_moe_layers > 0: + current_orth_loss = 0.0 + for module in self.modules(): + if isinstance(module, SVDMoeLinear): + if self.training_mode == 'hard_sampling': + current_orth_loss += module.compute_targeted_orthogonal_loss(self.active_expert_idx) + elif self.training_mode == 'standard': + current_orth_loss += module.compute_full_orthogonal_loss() + orth_loss = current_orth_loss / num_moe_layers + + # Balance loss is only relevant when the router is active + if self.training_mode == 'router_training' or self.training_mode == 'standard': + load_balancing_loss = pred_dict['balance_loss'] + + if self.training_mode == 'router_training' or self.training_mode == 'standard': + gating_logits = pred_dict['gating_logits'] + + if gating_logits is not None and expert_domain_labels is not None: + + valid_mask = (expert_domain_labels >= 0) & (expert_domain_labels < self.gating_network.num_experts) + + filtered_logits = gating_logits[valid_mask] + filtered_labels = expert_domain_labels[valid_mask] + + if filtered_logits.shape[0] > 0: + gating_classification_loss = self.gating_criterion(filtered_logits, filtered_labels) + + + total_loss = classification_loss + \ + self.moe_lambda_orth * orth_loss + \ + self.moe_lambda_balance * load_balancing_loss + \ + self.moe_lambda_gating_cls * gating_classification_loss + + return { + 'overall_loss': total_loss, + 'classification_loss': classification_loss.detach(), + 'orth_loss': orth_loss.detach(), + 'balance_loss': load_balancing_loss.detach(), + 'gating_cls_loss': gating_classification_loss.detach(), + } + + +class ViTMoEAttention(nn.Module): + def __init__(self, config: CLIPVisionConfig, num_experts: int, r_main: int, rank_per_expert: int, artifact_expert_idx: int, svd_dropout: float): + super().__init__() + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + + self.q_proj = SVDMoeLinear(self.embed_dim, self.embed_dim, r_main, num_experts, rank_per_expert, artifact_expert_idx, svd_dropout) + self.k_proj = SVDMoeLinear(self.embed_dim, self.embed_dim, r_main, num_experts, rank_per_expert, artifact_expert_idx, svd_dropout) + self.v_proj = SVDMoeLinear(self.embed_dim, self.embed_dim, r_main, num_experts, rank_per_expert, artifact_expert_idx, svd_dropout) + self.out_proj = SVDMoeLinear(self.embed_dim, self.embed_dim, r_main, num_experts, rank_per_expert, artifact_expert_idx, svd_dropout) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + gating_outputs: Dict[str, torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Input shape: Batch x Time x Channel + attention_mask: (batch, 1, a_seq_len, b_seq_len) + """ + bsz, tgt_len, embed_dim = hidden_states.size() + + query_states = self.q_proj(hidden_states, gating_outputs) * self.scale + key_states = self.k_proj(hidden_states, gating_outputs) + value_states = self.v_proj(hidden_states, gating_outputs) + + query_states = self._shape(query_states, tgt_len, bsz) + key_states = self._shape(key_states, -1, bsz) + value_states = self._shape(value_states, -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = query_states.reshape(*proj_shape) + key_states = key_states.reshape(*proj_shape) + value_states = value_states.reshape(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + attn_weights_reshaped = None + if output_attentions: + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, embed_dim) + + attn_output = self.out_proj(attn_output, gating_outputs) + + return attn_output, attn_weights_reshaped + + +class ViTMoELayer(nn.Module): + def __init__(self, config: CLIPVisionConfig, num_experts: int, r_main: int, rank_per_expert: int, artifact_expert_idx: int, svd_dropout: float): + super().__init__() + self.self_attn = ViTMoEAttention(config, num_experts, r_main, rank_per_expert, artifact_expert_idx, svd_dropout) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.mlp = CLIPMLP(config) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor, gating_outputs: Dict[str, torch.Tensor], attention_mask: Optional[torch.Tensor] = None): + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + + # Pass attention_mask to self_attn + hidden_states, attn_weights = self.self_attn( + hidden_states=hidden_states, + gating_outputs=gating_outputs, + attention_mask=attention_mask + ) + + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + # Maintain the return value format consistent with the official EncoderLayer (tuple) + return (hidden_states, ) + + +class SVDMoeLinear(nn.Module): + """ + SVD orthogonal subspace linear layer supporting multiple experts and dynamic gating. + """ + def __init__(self, in_features, out_features, r_main, num_experts, rank_per_expert, artifact_expert_idx, svd_dropout=0.0, bias=True): + super(SVDMoeLinear, self).__init__() + self.in_features = in_features + self.out_features = out_features + self.r_main = r_main + self.num_experts = num_experts + self.rank_per_expert = rank_per_expert + self.artifact_expert_idx = artifact_expert_idx + + self.register_buffer('weight_main', torch.zeros(out_features, in_features)) + self.register_buffer('U_r', torch.zeros(out_features, r_main)) + self.register_buffer('V_r', torch.zeros(r_main, in_features)) + + self.U_experts = nn.ParameterList([nn.Parameter(torch.zeros(out_features, rank_per_expert)) for _ in range(num_experts)]) + self.S_experts = nn.ParameterList([nn.Parameter(torch.zeros(rank_per_expert)) for _ in range(num_experts)]) + self.V_experts = nn.ParameterList([nn.Parameter(torch.zeros(rank_per_expert, in_features)) for _ in range(num_experts)]) + + self.register_buffer('weight_original_fnorm', torch.tensor(0.0)) + + if bias: + self.bias = nn.Parameter(torch.Tensor(out_features)) + nn.init.zeros_(self.bias) + else: + self.register_parameter('bias', None) + + if svd_dropout > 0.0: + self.dropout = nn.Dropout(p=svd_dropout) + else: + self.dropout = nn.Identity() + + def forward(self, x: torch.Tensor, gating_outputs: Dict[str, torch.Tensor]) -> torch.Tensor: + output_main = F.linear(x, self.weight_main, None) + + x = self.dropout(x) + + top_k_indices = gating_outputs['top_k_indices'] # Shape: [B, k] + top_k_gates = gating_outputs['top_k_gates'] # Shape: [B, k] + k = top_k_indices.size(1) + + expert_output = torch.zeros_like(output_main) + U_all = torch.stack([p for p in self.U_experts]) + S_all = torch.stack([p for p in self.S_experts]) + V_all = torch.stack([p for p in self.V_experts]) + + # Record the original input dimension (2 or 3) + original_dim = x.dim() + + # We use torch.bmm which requires 3D inputs + # If x is 2D (Batch, In_features), we unsqueeze it to 3D (Batch, 1, In_features) + if original_dim == 2: + x = x.unsqueeze(1) # Shape: [B, I] -> [B, 1, I] + # We do the same for expert_output to match dimensions for accumulation + expert_output = expert_output.unsqueeze(1) # Shape: [B, O] -> [B, 1, O] + + # Loop over the k choices + for i in range(k): + chosen_expert_indices = top_k_indices[:, i] # Shape: [B] + gate_values = top_k_gates[:, i].unsqueeze(-1) # Shape: [B, 1] + + U_batch = U_all[chosen_expert_indices] # Shape: [B, O, r] + S_batch = S_all[chosen_expert_indices] # Shape: [B, r] + V_batch = V_all[chosen_expert_indices] # Shape: [B, r, I] + + # 1. (x @ V.T) + # Shape: [B, S, I] @ [B, I, r] -> [B, S, r] + x_v = torch.bmm(x, V_batch.transpose(1, 2)) + + # 2. (x @ V.T) * S + # This is equivalent to (x @ V.T) @ diag(S) + # Shape: [B, S, r] * [B, 1, r] (broadcast) -> [B, S, r] + x_v_s = x_v * S_batch.unsqueeze(1) + + # 3. ((x @ V.T) * S) @ U.T is equivalent to `output = x @ (U @ diag(S) @ V).T` + # Shape: [B, S, r] @ [B, r, O] -> [B, S, O] + current_expert_output = torch.bmm(x_v_s, U_batch.transpose(1, 2)) + + # 4. Apply gate weights + # Shape: [B, S, O] * [B, 1, 1] (broadcast) -> [B, S, O] + expert_output += current_expert_output * gate_values.unsqueeze(-1) + + # If the input was 2D, squeeze the sequence dimension (S=1) back out + if original_dim == 2: + expert_output = expert_output.squeeze(1) # Shape: [B, 1, O] -> [B, O] + + final_output = output_main + expert_output + if self.bias is not None: + final_output = final_output + self.bias + return final_output + + def _calculate_pairwise_loss(self, base1, base2): + error = base1.t() @ base2 + return torch.norm(error, p='fro') + + def compute_targeted_orthogonal_loss(self, active_expert_idx: int) -> torch.Tensor: + """ + Efficient orthogonal loss for the hard sampling stage. + - If the active expert is the artifact expert, computes loss only against the main space. + - If the active expert is a semantic expert, computes loss against the main space + and previously trained *semantic* experts, skipping the artifact expert. + """ + loss = torch.tensor(0.0, device=self.weight_main.device) + num_pairs = 0 + + active_U = F.normalize(self.U_experts[active_expert_idx], dim=0) + active_V_t = F.normalize(self.V_experts[active_expert_idx], dim=1).t() + + # Loss vs. Main space + loss += self._calculate_pairwise_loss(self.U_r, active_U) + loss += self._calculate_pairwise_loss(self.V_r.t(), active_V_t) + num_pairs += 1 + + if active_expert_idx == self.artifact_expert_idx: + return loss / num_pairs + + # Loss vs. previously trained semantic experts + for i in range(active_expert_idx): + # Skip comparison with the artifact expert + if i == self.artifact_expert_idx: + continue + + prev_U = F.normalize(self.U_experts[i].detach(), dim=0) + prev_V_t = F.normalize(self.V_experts[i].detach(), dim=1).t() + loss += self._calculate_pairwise_loss(prev_U, active_U) + loss += self._calculate_pairwise_loss(prev_V_t, active_V_t) + num_pairs += 1 + + return loss / num_pairs if num_pairs > 0 else torch.tensor(0.0, device=self.weight_main.device) + + def compute_full_orthogonal_loss(self) -> torch.Tensor: + """ + Computes the full pairwise orthogonal loss between all components. + Used for end-to-end finetuning stages. + """ + all_U_bases = [self.U_r] + [F.normalize(u, dim=0) for u in self.U_experts] + all_V_bases_t = [self.V_r.t()] + [F.normalize(v, dim=1).t() for v in self.V_experts] + + loss = torch.tensor(0.0, device=self.weight_main.device) + num_pairs = 0 + + # Iterate over all unique pairs of basis matrices + for i in range(len(all_U_bases)): + for j in range(i + 1, len(all_U_bases)): + loss += self._calculate_pairwise_loss(all_U_bases[i], all_U_bases[j]) + loss += self._calculate_pairwise_loss(all_V_bases_t[i], all_V_bases_t[j]) + num_pairs += 1 + + return loss / num_pairs if num_pairs > 0 else torch.tensor(0.0, device=self.weight_main.device) + + +class GatingNetwork(nn.Module): + def __init__(self, input_dim: int, num_experts: int, hidden_dim: int = 256, top_k: int = 2): + super(GatingNetwork, self).__init__() + self.num_experts = num_experts + self.top_k = top_k + self.network = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, num_experts) + ) + + def forward(self, x: torch.Tensor) -> Dict[str, torch.Tensor]: + logits = self.network(x) # Shape: [B, N] + + top_k_logits, top_k_indices = torch.topk(logits, self.top_k, dim=-1) # Shape: [B, k] + top_k_gates = F.softmax(top_k_logits, dim=-1) # Shape: [B, k] + + router_probs = F.softmax(logits, dim=-1) + sparse_mask = torch.zeros_like(logits).scatter_(-1, top_k_indices, 1.0) + tokens_per_expert = torch.mean(sparse_mask.float(), dim=0) + router_prob_per_expert = torch.mean(router_probs, dim=0) + load_balancing_loss = self.num_experts * torch.sum(tokens_per_expert * router_prob_per_expert) + + return { + 'gating_logits': logits, + 'top_k_indices': top_k_indices, # The indices of the chosen experts + 'top_k_gates': top_k_gates, # The weights for the chosen experts + 'balance_loss': load_balancing_loss + } \ No newline at end of file diff --git a/models/image/omniaid/omniaid.py b/models/image/omniaid/omniaid.py new file mode 100644 index 0000000000000000000000000000000000000000..a321fdead322356f0db8d9a4153da790d7ba7e25 --- /dev/null +++ b/models/image/omniaid/omniaid.py @@ -0,0 +1,46 @@ +"""OmniAID (Decoupling Semantics and Artifacts for Universal AI-Generated Image Detection in the Wild, arXiv 2511.08423). + +Official repo: github.com/yunncheng/OmniAID (MIT). A CLIP ViT-L/14@336 whose attention projections are split by SVD into +a shared main subspace plus small expert subspaces (a hybrid mixture of experts): + + route = frozen CLIP vision tower (pooler output) → router → top-k SEMANTIC experts (e.g. human/animal, object/scene) + tokens = MoE ViT: every layer uses the routed semantic experts + the always-on ARTIFACT expert + logits = Linear(ln_post(CLS)) → [B, 2]; P(fake) = softmax(logits)[:, 1] (the official ``prob``) + +``SynthScopeOmniAIDModel`` *is* the official ``OmniAID`` class (vendored unchanged in ``official_arch.py``); the released +checkpoints hold the full model and load key-for-key with ``strict=True`` as in the official eval. The constructor, as in +the official code, first builds the experts from the CLIP weights (``openai/clip-vit-large-patch14-336``), which the +checkpoint then overwrites. Each release has its own expert configuration (number of experts, rank, router width, +top-k), set per checkpoint in the model configs. + +Training: the official recipe trains in two stages (each semantic expert with hard routing, then the router with the +artifact expert fixed) with orthogonality, load-balancing and gating losses; ``compute_loss`` here is only the +classification loss and does NOT reproduce that recipe. +""" +import types + +import torch.nn as nn +from omegaconf import DictConfig + +from .official_arch import OmniAID + +CONFIG_KEYS = ("CLIP_path", "num_experts", "rank_per_expert", "moe_lambda_orth", "moe_lambda_balance", + "moe_lambda_gating_cls", "moe_router_hidden_dim", "moe_top_k", "dropout", + "gradient_checkpointing_enable", "is_hybrid") + + +class SynthScopeOmniAIDModel(OmniAID): + """The official OmniAID model, configured from SynthScope's ModelConfig.""" + + def __init__(self, config: DictConfig): + mc = config.ModelConfig + super().__init__(types.SimpleNamespace(**{k: mc[k] for k in CONFIG_KEYS})) + self.criterion = nn.CrossEntropyLoss() + + def forward(self, images): + return super().forward(images)["cls"] + + def compute_loss(self, batch): + images, labels, _folder_ids = batch + logits = self.forward(images) + return {"loss": self.criterion(logits, labels), "outputs": logits} diff --git a/models/image/pgc/README.md b/models/image/pgc/README.md index 83509c08a36d5af88e4251fc48f2650acd64c10b..5f24520048b2cde14b699b51b2edd4632d2ea041 100644 --- a/models/image/pgc/README.md +++ b/models/image/pgc/README.md @@ -58,9 +58,9 @@ The residual is resolution-sensitive, so the experiment uses **crop-only** trans ## Training setup ```bash -conf/model/image/pgc.yaml # model_type: pgc, backbone_name, lora_*, tau_rgb/res -conf/trainer/image/pgc.yaml # Adam, lr 5e-5, weight_decay 0.05 -experiments/image/pgc.sh # trains on ProGAN (CNNDetection); swap dataset for GenImage SDv1.4 +conf/model/image/pgc.yaml # model_type: pgc, backbone_name, lora_*, tau_rgb/res, label smoothing 0.1 +conf/trainer/image/pgc.yaml # AdamW 5e-5, wd 0.05, 32 x 4 accumulation, clip 1.0, step budget (checkpoint every 100) +experiments/image/pgc.sh # the official ProGAN recipe (all 20 classes); see "Training in SynthScope" below ``` Registered in `trainer.py` under `model_type == "pgc"`. Trainable params: LoRA adapters + residual @@ -68,6 +68,37 @@ stream + PGCM + classifier (DINOv2 backbone frozen). The paper's benchmarks are AIGI, and UniversalFakeDetect; CommGen15 (the paper's new dataset) is optional — the SynthScope integration trains on the existing ProGAN/GenImage data. +## Training in SynthScope + +`experiments/image/pgc.sh` runs the official ProGAN recipe (`train.py` + `engine/trainer.py`, README 3.2) through +SynthScope's trainer: all 20 ProGAN classes (720k images, = the UniversalFakeDetect train set); flip → ColorJitter +(0.15, 0.15, 0.15, 0.05) p 0.2 → pad + RandomCrop 224 → ImageNet normalization, residual computed in the model; AdamW +5e-5, weight decay 0.05, label smoothing 0.1, micro-batch 32 × 4 accumulation, gradient clipping 1.0, seed 42, one GPU. +The authors' released checkpoint is only **~400 steps** in, because DINOv2's prior overfits ProGAN within the first +epoch, so the run is a budget of 2000 micro-batches (trainer `step_budget`) with a checkpoint every 100, and the early +curve is evaluated. (The official cosine schedule spans 100 epochs and is stepped per optimizer step, so the learning +rate stays at 5e-5 here.) The run takes 38 minutes on one A5000 (job 266276). + +ForenSynths with the same `bench_eval` command for every checkpoint (job 266282; FS-8 = the paper's 8 GANs, hard-3 = +deepfake, SAN, SeeingDark): + +| checkpoint | FS-8 mAP | FS-8 acc | hard-3 AUROC | FS-13 mAP | +|---|---|---|---|---| +| official `PGC_train_progan_ckpt.pth` | 96.41 | 76.34 | 90.87 | 93.80 | +| step 300 | 97.60 | 90.82 | 92.88 | 95.07 | +| **step 500** | **97.41** | **85.89** | **93.93** | **95.78** | +| step 900 | 97.82 | 85.71 | 93.88 | 97.12 | +| step 1300 | 97.63 | 85.61 | 92.76 | 97.03 | +| step 1700 | 97.21 | 84.06 | 91.03 | 96.69 | +| step 1900 | — | 74.49 (FS-13) | — | 95.97 | + +The paper reports no standalone ForenSynths number, so the reference is the official checkpoint: every checkpoint from +step 300 on matches or exceeds it on ForenSynths-8 mAP (96.41 on this evaluation); transfer to the hard generators peaks around steps 500-900 and then erodes, as the authors +describe. **Step 500**, the saved checkpoint nearest the authors' ~400 steps (picked by that rule, not by test score), +is the SynthScope-trained number in the table below; on UniversalFakeDetect, which the paper does report, it gives 98.99 mAP / 91.96 acc (paper +98.0 — reproduced; official 99.17 / 83.57 on the same run, job 266660). The GenImage, AIGIBench and CommGen15 SynthScope_train entries are +from an earlier SynthScope recipe (2 GPUs, full cosine over epochs) and have not been rerun with this one. + ## Reproduced benchmark results All benchmarks the paper reports, with our numbers (verified, traced to `repro/bench_results/` + `repro/benchmark_specs/`). @@ -75,14 +106,14 @@ All benchmarks the paper reports, with our numbers (verified, traced to `repro/b | Benchmark | Paper | SynthScope_official | SynthScope_train | Status | |---|---|---|---|---| | GenImage | map=100.0 / acc=98.3 | mAP=100.0 / acc=99.49 | mAP=97.62 / acc=78.6 | official | -| UniversalFakeDetect (UFD) | map=98.0 | mAP=99.34 / acc=94.16 | mAP=99.01 | reproduced | -| ForenSynths-8 (CNNDetection test) | mAP=96.25 | mAP=96.39 / acc=76.29 | mAP=96.05 / acc=71.12 | reproduced | +| UniversalFakeDetect (UFD) | map=98.0 | mAP=99.34 / acc=94.16 | **mAP=98.99 / acc=91.96** | **reproduced** (step 500) | +| ForenSynths-8 (CNNDetection test) | — (not reported separately; official checkpoint 96.25) | mAP=96.39 / acc=76.29 | **mAP=97.41 / acc=85.89** | matches/exceeds the official checkpoint (step 500) | | AIGIBench (AIGI) | map=95.0 / acc=86.6 | mAP=95.36 / acc=88.33 | mAP=86.88 | official | | CommGen15 (commercial in-the-wild) | acc=97.9 | mAP=99.26 / acc=97.39 | mAP=98.61 / acc=66.66 | official | > **ForenSynths hard generators — reproduced via early-stopping (resolved 2026-06-19).** The released ProGAN checkpoint is trained for only **~400 steps (< 1 epoch)** — confirmed by the authors on our issue. DINOv2's strong priors overfit ProGAN within a few epochs (loss → floor), which keeps easy generators high but **erodes hard-generator transfer**; our earlier epoch-level checkpoints (ep10+) all overshot the peak. Re-running the official `train.py` verbatim with fine-grained eval shows the hard-3 (deepfake/san/seeingdark) **mean AUROC climb to 94.56 at step ~900 ≈ official 94.35**, then decay with longer training. So PGC **does** reproduce from the published recipe+data with early-stopping inside the first epoch. (Served PGC uses the official weights.) See `repro/investigate/pgc_gap_investigation.md`. -> **Eval transform — `PadCenterCrop(224)`, no resize (verified 2026-06-19).** PGC's YCbCr residual is resolution-sensitive, so the eval transform is center-crop-224 with **no resize** (= the official `PadCenterCrop`; byte-identical to the genuine repo's). The earlier `Resize→CenterCrop` registry transform wiped the forensic signal on super-res/low-light generators (san 73.3, seeingdark 72.5 — same eval-transform family as the SAFE/AIDE/SPAI fixes). With the fix, SynthScope `bench_eval` reproduces ForenSynths-8 (96.39 mAP official / 96.05 from-scratch vs paper 96.25) and san recovers to 98.65 AUROC. One generator, **seeingdark**, stays ~10 pt below the genuine repo (77.9 vs 87.49) on identical images/weights with now-bit-exact preprocessing — isolated to the model forward, not the transform; it is not part of the reported ForenSynths-8. +> **Eval transform — `PadCenterCrop(224)`, no resize (verified 2026-06-19).** PGC's YCbCr residual is resolution-sensitive, so the eval transform is center-crop-224 with **no resize** (= the official `PadCenterCrop`; byte-identical to the genuine repo's). The earlier `Resize→CenterCrop` registry transform wiped the forensic signal on super-res/low-light generators (san 73.3, seeingdark 72.5 — same eval-transform family as the SAFE/AIDE/SPAI fixes). With the fix, SynthScope `bench_eval` gives ForenSynths-8 96.39 mAP with the official weights and 96.05 with the official code retrained (the paper reports no ForenSynths number; 96.25 was the earlier official-weights measurement) and san recovers to 98.65 AUROC. One generator, **seeingdark**, stays ~10 pt below the genuine repo (77.9 vs 87.49) on identical images/weights with now-bit-exact preprocessing — isolated to the model forward, not the transform; it is not part of the reported ForenSynths-8. **reproduced** = SynthScope-trained checkpoint matches the paper · **official** = official weights loaded into the SynthScope class match (the SynthScope-trained number is shown alongside where we trained from scratch) · **below** = gap remains · **reference** = paper reports no comparable number · **missing** = not yet evaluated. @@ -92,4 +123,4 @@ All benchmarks the paper reports, with our numbers (verified, traced to `repro/b |---|---| | Residual built in-model (not in the dataset transform) | Keeps the SynthScope dataset/transform pipeline standard (3-channel in). Same residual math (BT.601 quantization, Eq. 1) — `_quantization_residual` snaps `round(x01*255)` to the uint8 grid so the residual is **bit-exact** to the official 6-channel transform despite the in-model ImageNet de-normalization (verified zero-diff, `repro/_residual_path_diff.py`). | | DINOv2-Large loaded via HF name | No local-checkpoint requirement; LoRA targets use the HF DINOv2 module names. | -| Reduced training schedule for the bundled checkpoint | The shipped checkpoint is trained for turnaround on a ProGAN subset; raise epochs / use GenImage for a full reproduction. | +| Training budget | The official checkpoint is ~400 steps in; `experiments/image/pgc.sh` saves every 100 micro-batches up to 2000 instead of training 100 epochs, which overfits ProGAN. | diff --git a/pyproject.toml b/pyproject.toml index e3c83c10b7899f0dfd0d54b9a38f4f70d452f59a..6a64f8c1fff79879505c44077019b6ebbb20e9b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "pytorch_wavelets>=1.3", "PyWavelets>=1.9", "opencv-python-headless", # CuRe preprocessing (JPEG round-trips, resizing) + "scipy", # CNNDetection-style blur augmentation (transformations.CNNDetAugment) "setuptools<81", # pytorch_wavelets (SAFE) still imports pkg_resources, removed in setuptools 81 ] diff --git a/repro/MASTER_RESULTS.jsonl b/repro/MASTER_RESULTS.jsonl index ca1842a94c95cbbff6e341291802ee73bd5ba64f..afe04565d5cf0d9fe04c08db8365207039bff551 100644 --- a/repro/MASTER_RESULTS.jsonl +++ b/repro/MASTER_RESULTS.jsonl @@ -296,3 +296,42 @@ {"date": "2026-10-26T09:48:10", "tag": "d3qe_cnndetection", "model": "d3qe", "ckpt": "registry-default", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 7, "means": {"acc": 83.45, "map": 92.23, "mauroc": 90.65}} {"date": "2026-10-26T10:02:32", "tag": "d3qe_genimage_full", "model": "d3qe", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", "n_generators": 6, "means": {"acc": 79.44, "map": 90.94, "mauroc": 90.49}} {"date": "2026-10-26T10:14:34", "tag": "d3qe_rrdataset", "model": "d3qe", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RRDataset/test", "n_generators": 3, "means": {"acc": 54.95, "map": 59.25, "mauroc": 55.11}} +{"date": "2026-10-26T21:42:06", "tag": "d3qe_trained_ep3_arforensics", "model": "d3qe", "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/snapshot_ep3_best.pth", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/ARForensics/test", "n_generators": 7, "means": {"acc": 81.11, "map": 90.77, "mauroc": 90.97}} +{"date": "2026-10-27T06:02:46", "tag": "d3qe_trained_arforensics", "model": "d3qe", "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/model_best.pth", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/ARForensics/test", "n_generators": 7, "means": {"acc": 79.83, "map": 91.73, "mauroc": 91.49}} +{"date": "2026-10-27T06:28:33", "tag": "d3qe_trained_cnndetection", "model": "d3qe", "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/model_best.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 7, "means": {"acc": 82.14, "map": 91.35, "mauroc": 90.29}} +{"date": "2026-10-27T06:42:56", "tag": "d3qe_trained_genimage_full", "model": "d3qe", "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/model_best.pth", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", "n_generators": 6, "means": {"acc": 77.01, "map": 90.11, "mauroc": 89.63}} +{"date": "2026-10-27T02:49:02", "tag": "npr_official_nprpth_forensynths8", "model": "npr", "ckpt": "registry-default", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 8, "means": {"acc": 92.3, "map": 94.22, "mauroc": 96.08}} +{"date": "2026-10-27T02:49:53", "tag": "npr_official_nprpth_ufd", "model": "npr", "ckpt": "registry-default", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 94.57, "map": 97.16, "mauroc": 97.68}} +{"date": "2026-10-27T02:53:54", "tag": "npr_trained_last_forensynths8", "model": "npr", "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_last.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 8, "means": {"acc": 84.09, "map": 93.83, "mauroc": 94.94}} +{"date": "2026-10-27T02:54:44", "tag": "npr_trained_last_ufd", "model": "npr", "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_last.pth", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 88.91, "map": 97.49, "mauroc": 97.23}} +{"date": "2026-10-27T02:58:43", "tag": "npr_trained_best_forensynths8", "model": "npr", "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_best.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 8, "means": {"acc": 84.3, "map": 91.81, "mauroc": 93.3}} +{"date": "2026-10-27T02:59:33", "tag": "npr_trained_best_ufd", "model": "npr", "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_best.pth", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 85.34, "map": 95.56, "mauroc": 95.3}} +{"date": "2026-10-27T03:25:16", "tag": "npr_official3090_forensynths8", "model": "npr", "ckpt": "/ivi/zfs/s0/original_homes/ssalehi/SynthScope_shared/npr_ref/model_epoch_last_3090.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 8, "means": {"acc": 91.83, "map": 92.81, "mauroc": 95.18}} +{"date": "2026-10-27T03:26:05", "tag": "npr_official3090_ufd", "model": "npr", "ckpt": "/ivi/zfs/s0/original_homes/ssalehi/SynthScope_shared/npr_ref/model_epoch_last_3090.pth", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 93.94, "map": 96.69, "mauroc": 97.26}} +{"date": "2026-10-27T13:08:53", "tag": "npr_officialcode_rerun_forensynths8", "model": "npr", "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_official_rerun/4class-resnet-car-cat-chair-horse2026_10_27_03_27_09/model_epoch_last.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 8, "means": {"acc": 81.47, "map": 92.72, "mauroc": 93.97}} +{"date": "2026-10-27T13:09:44", "tag": "npr_officialcode_rerun_ufd", "model": "npr", "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_official_rerun/4class-resnet-car-cat-chair-horse2026_10_27_03_27_09/model_epoch_last.pth", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 87.61, "map": 96.56, "mauroc": 96.43}} +{"date": "2026-10-27T18:34:34", "tag": "pgc_official_forensynths13", "model": "pgc", "ckpt": "registry-default", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 70.58, "map": 93.8, "mauroc": 94.37}} +{"date": "2026-10-27T19:07:32", "tag": "pgc_trained_step300_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step300.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 79.88, "map": 95.07, "mauroc": 95.66}} +{"date": "2026-10-27T19:39:25", "tag": "pgc_trained_step500_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step500.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 82.98, "map": 95.78, "mauroc": 96.15}} +{"date": "2026-10-27T20:11:21", "tag": "pgc_trained_step700_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step700.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 80.35, "map": 96.67, "mauroc": 96.6}} +{"date": "2026-10-27T20:43:13", "tag": "pgc_trained_step900_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step900.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 79.92, "map": 97.12, "mauroc": 97.01}} +{"date": "2026-10-27T21:15:31", "tag": "pgc_trained_step1100_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1100.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 75.44, "map": 96.74, "mauroc": 96.5}} +{"date": "2026-10-27T21:47:29", "tag": "pgc_trained_step1300_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1300.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 78.24, "map": 97.03, "mauroc": 96.8}} +{"date": "2026-10-27T22:19:20", "tag": "pgc_trained_step1500_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1500.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 77.2, "map": 96.85, "mauroc": 96.54}} +{"date": "2026-10-27T22:51:19", "tag": "pgc_trained_step1700_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1700.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 74.18, "map": 96.69, "mauroc": 96.32}} +{"date": "2026-10-27T23:23:08", "tag": "pgc_trained_step1900_forensynths13", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1900.pth", "benchmark": "forensynths", "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", "n_generators": 13, "means": {"acc": 74.49, "map": 95.97, "mauroc": 95.35}} +{"date": "2026-10-28T00:46:03", "tag": "pgc_official_ufd8", "model": "pgc", "ckpt": "registry-default", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 83.57, "map": 99.17, "mauroc": 99.2}} +{"date": "2026-10-28T00:50:55", "tag": "pgc_trained_step500_ufd8", "model": "pgc", "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step500.pth", "benchmark": "ufd", "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", "n_generators": 8, "means": {"acc": 91.96, "map": 98.99, "mauroc": 98.92}} +{"date": "2026-10-27T08:37:18", "tag": "omniaid_synthwildx", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/SynthWildX/test", "n_generators": 3, "means": {"acc": 86.02, "map": 95.52, "mauroc": 95.31}} +{"date": "2026-10-27T08:42:46", "tag": "omniaid_wildrf", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/WildRF/test", "n_generators": 3, "means": {"acc": 88.36, "map": 97.28, "mauroc": 96.91}} +{"date": "2026-10-27T08:44:32", "tag": "omniaid_bfree_online", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/BFreeOnline/test", "n_generators": 1, "means": {"acc": 70.26, "map": 91.7, "mauroc": 87.18}} +{"date": "2026-10-27T09:09:34", "tag": "omniaid_realchain", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RealChain/test", "n_generators": 7, "means": {"acc": 61.71, "map": 84.02, "mauroc": 83.63}} +{"date": "2026-10-27T09:20:42", "tag": "omniaid_genimage_genimage_full", "model": "omniaid_genimage", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", "n_generators": 8, "means": {"acc": 95.82, "map": 99.02, "mauroc": 99.16}} +{"date": "2026-10-27T09:22:13", "tag": "omniaid_v1_genimage_full", "model": "omniaid_v1", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", "n_generators": 8, "means": {"acc": 97.16, "map": 99.2, "mauroc": 99.12}} +{"date": "2026-10-27T11:05:31", "tag": "omniaid_evalgen", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/EvalGEN/test", "n_generators": 5, "means": {"acc": 74.14, "map": 96.46, "mauroc": 93.08}} +{"date": "2026-10-27T12:23:42", "tag": "omniaid_rrdataset", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RRDataset/test", "n_generators": 3, "means": {"acc": 71.75, "map": 84.19, "mauroc": 82.32}} +{"date": "2026-10-27T13:04:46", "tag": "omniaid_genimage_aigcdetect", "model": "omniaid_genimage", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/AIGCDetect/test", "n_generators": 17, "means": {"acc": 88.87, "map": 97.88, "mauroc": 98.2}} +{"date": "2026-10-27T13:11:33", "tag": "omniaid_v1_aigcdetect", "model": "omniaid_v1", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/AIGCDetect/test", "n_generators": 17, "means": {"acc": 92.88, "map": 98.93, "mauroc": 98.99}} +{"date": "2026-10-27T15:40:59", "tag": "omniaid_genimage_mirage_test", "model": "omniaid_genimage", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/MirageTest/test", "n_generators": 5, "means": {"acc": 54.66, "map": 50.6, "mauroc": 50.74}} +{"date": "2026-10-27T15:49:26", "tag": "omniaid_v1_mirage_test", "model": "omniaid_v1", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/MirageTest/test", "n_generators": 5, "means": {"acc": 88.63, "map": 97.04, "mauroc": 97.97}} +{"date": "2026-10-27T16:20:10", "tag": "omniaid_mirage_test", "model": "omniaid", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/MirageTest/test", "n_generators": 5, "means": {"acc": 80.37, "map": 96.48, "mauroc": 96.65}} diff --git a/repro/RESULTS_SUMMARY.md b/repro/RESULTS_SUMMARY.md index 9872e223f3e127d0fa94328d1e4446d431363653..11b9a414dcc578377c2cdcf86bf9aa813dc94fe9 100644 --- a/repro/RESULTS_SUMMARY.md +++ b/repro/RESULTS_SUMMARY.md @@ -11,17 +11,18 @@ | Method | Headline benchmark | Paper | SynthScope_official | SynthScope_train | Verdict | |---|---|---:|---:|---:|---| | FatFormer | ForenSynths-8 | 99.7 | 99.70 | — | ✅ official match | -| NPR | ForenSynths-8 / GANGen | 96.1 / 96.6 | 94.22 / 96.49 | — | ✅ official match | +| NPR | ForenSynths-8 / GANGen | 96.1 / 96.6 | 94.22 / 96.49 | 93.83 / — | ✅ official match; trained (official recipe): AP close, acc 84.1 — the official train.py rerun here gets 81.5 | | CLIP/UnivFD | ForenSynths | — | 93.14 | 92.99 | ✅ match | | CNNDetection | ForenSynths | 90.8 | 91.15 | 91.05 | ✅ match | | SAFE | GenImage | 99.3 | 99.32 | 99.51 | ✅ retrain reproduces | | AIDE | GenImage (SD-v1.4) | 86.9 acc | 97.12 | 96.42 | ✅ reproduced (native-res retrain) | | SPAI | GenImage | ~91 AUC | 94.22 | 88.80 | ✅ served native-res (beats vendored 91.33) | -| PGC | GenImage | 100 AP | 100.0 | 97.62 | ✅ near | +| PGC | GenImage / ForenSynths-8 / UFD | 100 / — / 98.0 AP | 100.0 / 96.41 / 99.17 | 97.62 / **97.41** / **98.99** | ✅ UFD reproduced by the official recipe in SynthScope (step 500); ForenSynths (not in the paper) above the official checkpoint | | VIB-Net | GenImage (SD-v1.4) | 95.5 AP | *none released* | 83.26 | ⚠️ recipe/arch gap | | Co-Spy | Co-Spy-Bench-110 | 96.0 AP | 86.65 | 96.60 | ✅ beats official | | DDA | AIGCDetect / ForenSynths / EvalGEN / SynthWildX / WildRF | 87.8 / 81.4 / 97.2 / 90.9 / 90.3 acc | 87.84 / 81.43 / 96.86 / 90.80 / 90.20 acc | — | ✅ official match (GenImage 2.3 pt below) | -| D3QE | ARForensics / ForenSynths-7 / GenImage-6 | 82.11 / 83.73 / 78.61 acc | 81.74 / 83.45 / 79.44 acc | — | ✅ official match | +| D3QE | ARForensics / ForenSynths-7 / GenImage-6 | 82.11 / 83.73 / 78.61 acc | 81.74 / 83.45 / 79.44 acc | 79.83 / 82.14 / 77.01 acc | ✅ official match; trained in SynthScope: AP matches, acc 1.6-2.3 below | +| OmniAID | GenImage / AIGCDetect / Mirage-Test (GenImage release) | 95.94 / 88.87 / 51.10 acc | 95.82 / 88.87 / 51.10 acc | — | ✅ official match (v1 release too: 97.16 / 92.88 / 88.39 vs 97.24 / 92.88 / 88.39) | | D3 (video) | GenVideo-Val | 98.5 AP | 97.93 (official code) | 99.14 | ✅ reproduced (D3 protocol; preprocessing matched to the official code) | | ReStraV (video) | GenVidBench | 93.0 | — | 92.70 | ✅ match | | STALL (video) | GenVideo-Val | ~80 AUC | — | 83.5 AUROC | ✅ match | @@ -46,12 +47,12 @@ SPAI now SERVES official weights remapped into the SynthScope class at NATIVE re | Benchmark | Paper | SynthScope_official | SynthScope_train | |---|---:|---:|---:| | GenImage-8 | 100 AP | 100.0 (99.5) | **97.62 (78.6)** | -| UniversalFakeDetect-8 | — | 99.34 (94.2) | 99.01 | -| ForenSynths-13 | — | 96.25 (83.7) | 94.57 (83.4) — see hard-3 caveat | +| UniversalFakeDetect-8 | 98.0 AP | 99.34 (94.2) | **98.99 (92.0)** | +| ForenSynths-13 | — | 96.25 (83.7) | **95.78** (FS-8 97.41 / 85.9) | | AIGIBench-25 | — | 95.36 (88.3) | 86.88 | | CommGen15-15 | — | 99.26 (97.4) | **98.61 (66.7)** | -*PGC ForenSynths caveat:* the 13-gen SynthScope_train mean (94.57) is close to official (96.25), but only because the 10 easy generators (~99) dominate. On the three hard generators (deepfake/san/seeingdark) SynthScope_train is **75.87 mAUROC vs 94.35** official. Running the authors' `train.py` verbatim reproduces the same plateau → the released checkpoint is not reproducible from the published recipe+data. See `repro/investigate/pgc_gap_investigation.md`. +*PGC training:* the released ProGAN checkpoint is only ~400 steps in (the authors, on our issue), and DINOv2's prior overfits ProGAN within the first epoch, so the earlier epoch-level retrains (GenImage, AIGIBench, CommGen15 rows above, and the old 94.57 / hard-3 75.87 ForenSynths entry) overshot the transfer peak. The official recipe run in SynthScope with a 2000-micro-batch budget (`experiments/image/pgc.sh`, checkpoint every 100) matches or exceeds the official checkpoint at every step from 300 on: step 500 (the saved step nearest ~400) gives FS-8 97.41 mAP and hard-3 AUROC 93.93 (official 96.41 / 90.87 on the same evaluation) and UFD 98.99 (paper 98.0); transfer peaks around steps 500-900 and erodes after. GenImage, AIGIBench and CommGen15 have not been rerun with this recipe. See `repro/investigate/pgc_gap_investigation.md` and `models/image/pgc/README.md`. ### Co-Spy | Benchmark | Paper | SynthScope_official | SynthScope_train | @@ -148,7 +149,22 @@ Fidelity: SynthScope's class == the official `D3QE` (871/871 state-dict tensors | GenImage, the paper's 6 subsets (`--dataset genimage_full --select ...`) | **79.44** | 90.94 | **78.61 / 89.60** | mean reproduced; per subset up to 6.8 pt off in both directions (ADM 63.7 vs 70.4, Wukong 90.7 vs 84.4), likely a different test sample | | In the wild: SynthWildX / WildRF / BFree-Online / RealChain / EvalGEN / RRDataset | 49.6 / 53.0 / 47.0 / 49.5 / 62.2 / 55.0 | – | — | new measurements: it calls nearly every recent fake real (AR-only training) | -**Verdict: faithful port (identical to the official model); all 3 paper tables reproduced on the mean** (ARForensics and ForenSynths per generator too). Note: in the released code the codebook-frequency bias is zero at inference (`freq_log_counter` is not saved), so the reported scores come from the quantization-error and CLIP branches. +**Verdict: faithful port (identical to the official model); all 3 paper tables reproduced on the mean** (ARForensics and ForenSynths per generator too). **Trained in SynthScope with the official recipe** (`experiments/image/d3qe.sh`, job 265064): 79.83 / 91.73, 82.14 / 91.35, 77.01 / 90.11 (acc / AP) — AP within 0.9 pt of the paper everywhere, accuracy 1.6-2.3 pt lower (a slightly more conservative threshold, not a weaker ranking). Note: in the released code the codebook-frequency bias is zero at inference (`freq_log_counter` is not saved), so the reported scores come from the quantization-error and CLIP branches. + +### OmniAID (official checkpoints) +Fidelity: SynthScope's class == the official `OmniAID` for all three releases (state dicts identical, logits identical on 24 GenImage images, 0 verdict flips; job 265851). Accuracy at 0.5, mean over subsets; Mirage-Test mean = 4 balanced categories + Anime fake accuracy (the paper's Table 3). + +| Benchmark | release | acc | mAP | Paper | Note | +|---|---|---:|---:|---|---| +| GenImage, 8 subsets (`--dataset genimage_full`) | `omniaid_genimage` | **95.82** | 99.02 | **95.94** | **reproduced**, every subset within 1.5 pt; confirms the v1-era 512 -> 336 preprocessing | +| AIGCDetect, 17 (`--dataset aigcdetect`) | `omniaid_genimage` | **88.87** | 97.88 | **88.87** | **reproduced** exactly | +| Mirage-Test, 5 categories (`--dataset mirage_test`) | `omniaid_genimage` | **51.10** | — | **51.10** | **reproduced** exactly, every category | +| GenImage, 8 subsets | `omniaid_v1` | **97.16** | 99.20 | **97.24** | **reproduced**, every subset within 0.6 pt | +| AIGCDetect, 17 | `omniaid_v1` | **92.88** | 98.93 | **92.88** | **reproduced** exactly | +| Mirage-Test, 5 categories | `omniaid_v1` | **88.39** | — | **88.39** | **reproduced** exactly | +| Mirage-Test / SynthWildX / WildRF / BFree-Online / RealChain / EvalGEN / RRDataset | `omniaid` (v2, served) | 78.46 / 86.02 / 88.36 / 70.26 / 61.71 / 74.14 / 71.75 | — | — | new measurements | + +**Verdict: faithful port; every paper number SynthScope can run is reproduced** (Chameleon and DRCT-2M are not available). ### Video detectors (no official weights released) | Method · Benchmark | Paper | SynthScope_train | diff --git a/repro/bench_results/d3qe_trained_arforensics.json b/repro/bench_results/d3qe_trained_arforensics.json new file mode 100644 index 0000000000000000000000000000000000000000..f042a791df011982b133587eb17a319042d0504f --- /dev/null +++ b/repro/bench_results/d3qe_trained_arforensics.json @@ -0,0 +1,67 @@ +{ + "tag": "d3qe_trained_arforensics", + "model": "d3qe", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/model_best.pth", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/ARForensics/test", + "jpeg_fakes": null, + "n_generators": 7, + "means": { + "acc": 79.83, + "map": 91.73, + "mauroc": 91.49 + }, + "per_generator": { + "Infinity": { + "ap": 78.61, + "auroc": 79.4, + "real_acc": 97.38, + "fake_acc": 20.68, + "acc": 59.03 + }, + "Janus_Pro": { + "ap": 97.31, + "auroc": 97.06, + "real_acc": 97.6, + "fake_acc": 82.52, + "acc": 90.06 + }, + "LlamaGen": { + "ap": 99.62, + "auroc": 99.66, + "real_acc": 97.8, + "fake_acc": 97.18, + "acc": 97.49 + }, + "Open_MAGVIT2": { + "ap": 86.39, + "auroc": 85.63, + "real_acc": 97.42, + "fake_acc": 39.08, + "acc": 68.25 + }, + "RAR": { + "ap": 97.71, + "auroc": 97.67, + "real_acc": 97.47, + "fake_acc": 83.62, + "acc": 90.54 + }, + "Switti": { + "ap": 87.59, + "auroc": 86.6, + "real_acc": 97.4, + "fake_acc": 44.62, + "acc": 71.01 + }, + "VAR": { + "ap": 94.89, + "auroc": 94.42, + "real_acc": 97.43, + "fake_acc": 67.42, + "acc": 82.43 + } + }, + "date": "2026-10-27T06:02:46", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/d3qe_trained_cnndetection.json b/repro/bench_results/d3qe_trained_cnndetection.json new file mode 100644 index 0000000000000000000000000000000000000000..0a981a081ab22fec4104e0d5741eee6245952531 --- /dev/null +++ b/repro/bench_results/d3qe_trained_cnndetection.json @@ -0,0 +1,67 @@ +{ + "tag": "d3qe_trained_cnndetection", + "model": "d3qe", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/model_best.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 7, + "means": { + "acc": 82.14, + "map": 91.35, + "mauroc": 90.29 + }, + "per_generator": { + "biggan": { + "ap": 94.39, + "auroc": 93.54, + "real_acc": 97.6, + "fake_acc": 69.65, + "acc": 83.63 + }, + "cyclegan": { + "ap": 93.08, + "auroc": 92.91, + "real_acc": 95.46, + "fake_acc": 66.92, + "acc": 81.19 + }, + "gaugan": { + "ap": 97.36, + "auroc": 97.86, + "real_acc": 97.02, + "fake_acc": 87.44, + "acc": 92.23 + }, + "progan": { + "ap": 97.37, + "auroc": 97.68, + "real_acc": 94.85, + "fake_acc": 92.98, + "acc": 93.91 + }, + "stargan": { + "ap": 83.68, + "auroc": 82.03, + "real_acc": 96.25, + "fake_acc": 50.18, + "acc": 73.21 + }, + "stylegan": { + "ap": 85.88, + "auroc": 82.53, + "real_acc": 95.19, + "fake_acc": 55.68, + "acc": 75.44 + }, + "stylegan2": { + "ap": 87.68, + "auroc": 85.46, + "real_acc": 95.82, + "fake_acc": 54.88, + "acc": 75.35 + } + }, + "date": "2026-10-27T06:28:33", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/d3qe_trained_ep3_arforensics.json b/repro/bench_results/d3qe_trained_ep3_arforensics.json new file mode 100644 index 0000000000000000000000000000000000000000..07f3cfe204f348132871f2d1d4988d17f566bb5b --- /dev/null +++ b/repro/bench_results/d3qe_trained_ep3_arforensics.json @@ -0,0 +1,67 @@ +{ + "tag": "d3qe_trained_ep3_arforensics", + "model": "d3qe", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/snapshot_ep3_best.pth", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/ARForensics/test", + "jpeg_fakes": null, + "n_generators": 7, + "means": { + "acc": 81.11, + "map": 90.77, + "mauroc": 90.97 + }, + "per_generator": { + "Infinity": { + "ap": 79.06, + "auroc": 80.41, + "real_acc": 94.67, + "fake_acc": 33.8, + "acc": 64.23 + }, + "Janus_Pro": { + "ap": 96.19, + "auroc": 96.36, + "real_acc": 95.4, + "fake_acc": 85.65, + "acc": 90.53 + }, + "LlamaGen": { + "ap": 98.59, + "auroc": 98.89, + "real_acc": 95.38, + "fake_acc": 95.78, + "acc": 95.58 + }, + "Open_MAGVIT2": { + "ap": 85.27, + "auroc": 84.87, + "real_acc": 95.13, + "fake_acc": 45.12, + "acc": 70.13 + }, + "RAR": { + "ap": 96.75, + "auroc": 97.04, + "real_acc": 94.87, + "fake_acc": 86.83, + "acc": 90.85 + }, + "Switti": { + "ap": 86.43, + "auroc": 86.26, + "real_acc": 94.68, + "fake_acc": 53.87, + "acc": 74.27 + }, + "VAR": { + "ap": 93.07, + "auroc": 92.93, + "real_acc": 95.3, + "fake_acc": 69.12, + "acc": 82.21 + } + }, + "date": "2026-10-26T21:42:06", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/d3qe_trained_genimage_full.json b/repro/bench_results/d3qe_trained_genimage_full.json new file mode 100644 index 0000000000000000000000000000000000000000..542f95c7605442b57b28156d12b44f3f196ad3f3 --- /dev/null +++ b/repro/bench_results/d3qe_trained_genimage_full.json @@ -0,0 +1,60 @@ +{ + "tag": "d3qe_trained_genimage_full", + "model": "d3qe", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/d3qe_v1/model_best.pth", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", + "jpeg_fakes": null, + "n_generators": 6, + "means": { + "acc": 77.01, + "map": 90.11, + "mauroc": 89.63 + }, + "per_generator": { + "adm": { + "ap": 77.51, + "auroc": 73.53, + "real_acc": 97.8, + "fake_acc": 27.53, + "acc": 62.67 + }, + "glide": { + "ap": 97.62, + "auroc": 98.08, + "real_acc": 97.67, + "fake_acc": 81.13, + "acc": 89.4 + }, + "midjourney": { + "ap": 80.15, + "auroc": 81.72, + "real_acc": 97.2, + "fake_acc": 25.93, + "acc": 61.57 + }, + "sdv1.4": { + "ap": 93.68, + "auroc": 92.81, + "real_acc": 97.72, + "fake_acc": 61.12, + "acc": 79.42 + }, + "sdv1.5": { + "ap": 94.38, + "auroc": 94.5, + "real_acc": 96.2, + "fake_acc": 67.13, + "acc": 81.67 + }, + "wukong": { + "ap": 97.3, + "auroc": 97.16, + "real_acc": 97.73, + "fake_acc": 76.93, + "acc": 87.33 + } + }, + "date": "2026-10-27T06:42:56", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_official3090_forensynths8.json b/repro/bench_results/npr_official3090_forensynths8.json new file mode 100644 index 0000000000000000000000000000000000000000..1354de700f8b8517819333e87597d1de5bbbc1fd --- /dev/null +++ b/repro/bench_results/npr_official3090_forensynths8.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_official3090_forensynths8", + "model": "npr", + "ckpt": "/ivi/zfs/s0/original_homes/ssalehi/SynthScope_shared/npr_ref/model_epoch_last_3090.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 91.83, + "map": 92.81, + "mauroc": 95.18 + }, + "per_generator": { + "biggan": { + "ap": 85.59, + "auroc": 90.86, + "real_acc": 69.75, + "fake_acc": 98.15, + "acc": 83.95 + }, + "cyclegan": { + "ap": 98.12, + "auroc": 98.86, + "real_acc": 96.9, + "fake_acc": 93.49, + "acc": 95.19 + }, + "deepfake": { + "ap": 76.06, + "auroc": 83.04, + "real_acc": 61.17, + "fake_acc": 93.25, + "acc": 77.21 + }, + "gaugan": { + "ap": 82.97, + "auroc": 88.99, + "real_acc": 62.72, + "fake_acc": 99.16, + "acc": 80.94 + }, + "progan": { + "ap": 99.98, + "auroc": 99.97, + "real_acc": 99.9, + "fake_acc": 99.9, + "acc": 99.9 + }, + "stargan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.8, + "fake_acc": 99.75, + "acc": 99.77 + }, + "stylegan": { + "ap": 99.78, + "auroc": 99.77, + "real_acc": 99.93, + "fake_acc": 96.19, + "acc": 98.06 + }, + "stylegan2": { + "ap": 99.95, + "auroc": 99.97, + "real_acc": 99.95, + "fake_acc": 99.26, + "acc": 99.61 + } + }, + "date": "2026-10-27T03:25:16", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_official3090_ufd.json b/repro/bench_results/npr_official3090_ufd.json new file mode 100644 index 0000000000000000000000000000000000000000..5b63416bf2eda03762f164300b57a97d2535fc30 --- /dev/null +++ b/repro/bench_results/npr_official3090_ufd.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_official3090_ufd", + "model": "npr", + "ckpt": "/ivi/zfs/s0/original_homes/ssalehi/SynthScope_shared/npr_ref/model_epoch_last_3090.pth", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 93.94, + "map": 96.69, + "mauroc": 97.26 + }, + "per_generator": { + "dalle": { + "ap": 98.08, + "auroc": 98.08, + "real_acc": 98.0, + "fake_acc": 83.8, + "acc": 90.9 + }, + "glide_100_10": { + "ap": 99.52, + "auroc": 99.54, + "real_acc": 98.0, + "fake_acc": 97.6, + "acc": 97.8 + }, + "glide_100_27": { + "ap": 99.53, + "auroc": 99.61, + "real_acc": 98.0, + "fake_acc": 96.7, + "acc": 97.35 + }, + "glide_50_27": { + "ap": 99.55, + "auroc": 99.61, + "real_acc": 98.0, + "fake_acc": 96.9, + "acc": 97.45 + }, + "guided": { + "ap": 78.12, + "auroc": 82.46, + "real_acc": 62.9, + "fake_acc": 85.0, + "acc": 73.95 + }, + "ldm_100": { + "ap": 99.57, + "auroc": 99.61, + "real_acc": 98.0, + "fake_acc": 97.9, + "acc": 97.95 + }, + "ldm_200": { + "ap": 99.63, + "auroc": 99.69, + "real_acc": 98.0, + "fake_acc": 98.3, + "acc": 98.15 + }, + "ldm_200_cfg": { + "ap": 99.49, + "auroc": 99.48, + "real_acc": 98.0, + "fake_acc": 97.9, + "acc": 97.95 + } + }, + "date": "2026-10-27T03:26:05", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_official_nprpth_forensynths8.json b/repro/bench_results/npr_official_nprpth_forensynths8.json new file mode 100644 index 0000000000000000000000000000000000000000..cb4b24307f604c2819ac1ebc7c53d02f50ecd103 --- /dev/null +++ b/repro/bench_results/npr_official_nprpth_forensynths8.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_official_nprpth_forensynths8", + "model": "npr", + "ckpt": "registry-default", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 92.3, + "map": 94.22, + "mauroc": 96.08 + }, + "per_generator": { + "biggan": { + "ap": 87.8, + "auroc": 92.36, + "real_acc": 70.0, + "fake_acc": 98.7, + "acc": 84.35 + }, + "cyclegan": { + "ap": 98.45, + "auroc": 99.04, + "real_acc": 96.06, + "fake_acc": 96.14, + "acc": 96.1 + }, + "deepfake": { + "ap": 82.4, + "auroc": 86.85, + "real_acc": 70.52, + "fake_acc": 89.96, + "acc": 80.24 + }, + "gaugan": { + "ap": 85.49, + "auroc": 90.75, + "real_acc": 65.54, + "fake_acc": 99.46, + "acc": 82.5 + }, + "progan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.83, + "fake_acc": 99.75, + "acc": 99.79 + }, + "stargan": { + "ap": 99.94, + "auroc": 99.96, + "real_acc": 98.75, + "fake_acc": 99.95, + "acc": 99.35 + }, + "stylegan": { + "ap": 99.78, + "auroc": 99.75, + "real_acc": 99.65, + "fake_acc": 95.76, + "acc": 97.7 + }, + "stylegan2": { + "ap": 99.94, + "auroc": 99.94, + "real_acc": 99.74, + "fake_acc": 97.02, + "acc": 98.38 + } + }, + "date": "2026-10-27T02:49:02", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_official_nprpth_ufd.json b/repro/bench_results/npr_official_nprpth_ufd.json new file mode 100644 index 0000000000000000000000000000000000000000..d35785c4e5a726420d7dd5c2a55797a99659f6b6 --- /dev/null +++ b/repro/bench_results/npr_official_nprpth_ufd.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_official_nprpth_ufd", + "model": "npr", + "ckpt": "registry-default", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 94.57, + "map": 97.16, + "mauroc": 97.68 + }, + "per_generator": { + "dalle": { + "ap": 98.96, + "auroc": 98.96, + "real_acc": 98.5, + "fake_acc": 89.2, + "acc": 93.85 + }, + "glide_100_10": { + "ap": 99.63, + "auroc": 99.64, + "real_acc": 98.5, + "fake_acc": 96.7, + "acc": 97.6 + }, + "glide_100_27": { + "ap": 99.55, + "auroc": 99.56, + "real_acc": 98.5, + "fake_acc": 95.7, + "acc": 97.1 + }, + "glide_50_27": { + "ap": 99.61, + "auroc": 99.62, + "real_acc": 98.5, + "fake_acc": 96.6, + "acc": 97.55 + }, + "guided": { + "ap": 80.28, + "auroc": 84.23, + "real_acc": 65.8, + "fake_acc": 84.4, + "acc": 75.1 + }, + "ldm_100": { + "ap": 99.8, + "auroc": 99.87, + "real_acc": 98.5, + "fake_acc": 98.8, + "acc": 98.65 + }, + "ldm_200": { + "ap": 99.76, + "auroc": 99.82, + "real_acc": 98.5, + "fake_acc": 98.4, + "acc": 98.45 + }, + "ldm_200_cfg": { + "ap": 99.73, + "auroc": 99.76, + "real_acc": 98.5, + "fake_acc": 98.1, + "acc": 98.3 + } + }, + "date": "2026-10-27T02:49:53", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_officialcode_rerun_forensynths8.json b/repro/bench_results/npr_officialcode_rerun_forensynths8.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa7958fd034a4ac25ce45fabb34cdc5cd6f31d8 --- /dev/null +++ b/repro/bench_results/npr_officialcode_rerun_forensynths8.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_officialcode_rerun_forensynths8", + "model": "npr", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_official_rerun/4class-resnet-car-cat-chair-horse2026_10_27_03_27_09/model_epoch_last.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 81.47, + "map": 92.72, + "mauroc": 93.97 + }, + "per_generator": { + "biggan": { + "ap": 90.16, + "auroc": 91.79, + "real_acc": 91.95, + "fake_acc": 65.2, + "acc": 78.58 + }, + "cyclegan": { + "ap": 97.81, + "auroc": 98.64, + "real_acc": 99.24, + "fake_acc": 45.19, + "acc": 72.22 + }, + "deepfake": { + "ap": 73.75, + "auroc": 75.52, + "real_acc": 95.57, + "fake_acc": 20.53, + "acc": 58.05 + }, + "gaugan": { + "ap": 80.85, + "auroc": 86.74, + "real_acc": 90.64, + "fake_acc": 42.6, + "acc": 66.62 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 100.0, + "fake_acc": 99.87, + "acc": 99.94 + }, + "stargan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 100.0, + "fake_acc": 81.54, + "acc": 90.77 + }, + "stylegan": { + "ap": 99.34, + "auroc": 99.23, + "real_acc": 99.98, + "fake_acc": 83.07, + "acc": 91.53 + }, + "stylegan2": { + "ap": 99.89, + "auroc": 99.87, + "real_acc": 99.99, + "fake_acc": 88.17, + "acc": 94.08 + } + }, + "date": "2026-10-27T13:08:53", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_officialcode_rerun_ufd.json b/repro/bench_results/npr_officialcode_rerun_ufd.json new file mode 100644 index 0000000000000000000000000000000000000000..de9064dce40b4da84751ad4251b71fd02285c65c --- /dev/null +++ b/repro/bench_results/npr_officialcode_rerun_ufd.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_officialcode_rerun_ufd", + "model": "npr", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_official_rerun/4class-resnet-car-cat-chair-horse2026_10_27_03_27_09/model_epoch_last.pth", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 87.61, + "map": 96.56, + "mauroc": 96.43 + }, + "per_generator": { + "dalle": { + "ap": 92.55, + "auroc": 92.27, + "real_acc": 99.1, + "fake_acc": 38.4, + "acc": 68.75 + }, + "glide_100_10": { + "ap": 99.17, + "auroc": 99.15, + "real_acc": 99.1, + "fake_acc": 89.6, + "acc": 94.35 + }, + "glide_100_27": { + "ap": 98.95, + "auroc": 98.95, + "real_acc": 99.1, + "fake_acc": 84.9, + "acc": 92.0 + }, + "glide_50_27": { + "ap": 99.0, + "auroc": 98.94, + "real_acc": 99.1, + "fake_acc": 87.0, + "acc": 93.05 + }, + "guided": { + "ap": 85.77, + "auroc": 85.06, + "real_acc": 89.5, + "fake_acc": 56.5, + "acc": 73.0 + }, + "ldm_100": { + "ap": 99.03, + "auroc": 99.1, + "real_acc": 99.1, + "fake_acc": 87.8, + "acc": 93.45 + }, + "ldm_200": { + "ap": 99.08, + "auroc": 99.12, + "real_acc": 99.1, + "fake_acc": 87.5, + "acc": 93.3 + }, + "ldm_200_cfg": { + "ap": 98.95, + "auroc": 98.85, + "real_acc": 99.1, + "fake_acc": 86.9, + "acc": 93.0 + } + }, + "date": "2026-10-27T13:09:44", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_trained_best_forensynths8.json b/repro/bench_results/npr_trained_best_forensynths8.json new file mode 100644 index 0000000000000000000000000000000000000000..19e40afc6a26ed044a3a6668dda905047d0bb8ba --- /dev/null +++ b/repro/bench_results/npr_trained_best_forensynths8.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_trained_best_forensynths8", + "model": "npr", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_best.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 84.3, + "map": 91.81, + "mauroc": 93.3 + }, + "per_generator": { + "biggan": { + "ap": 89.71, + "auroc": 90.53, + "real_acc": 90.3, + "fake_acc": 68.7, + "acc": 79.5 + }, + "cyclegan": { + "ap": 97.55, + "auroc": 98.21, + "real_acc": 98.49, + "fake_acc": 66.62, + "acc": 82.55 + }, + "deepfake": { + "ap": 69.88, + "auroc": 74.96, + "real_acc": 70.37, + "fake_acc": 66.53, + "acc": 68.45 + }, + "gaugan": { + "ap": 78.39, + "auroc": 83.97, + "real_acc": 89.12, + "fake_acc": 44.8, + "acc": 66.96 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 100.0, + "fake_acc": 99.65, + "acc": 99.83 + }, + "stargan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 100.0, + "fake_acc": 75.49, + "acc": 87.74 + }, + "stylegan": { + "ap": 99.04, + "auroc": 98.85, + "real_acc": 99.93, + "fake_acc": 84.51, + "acc": 92.22 + }, + "stylegan2": { + "ap": 99.9, + "auroc": 99.88, + "real_acc": 99.94, + "fake_acc": 94.29, + "acc": 97.11 + } + }, + "date": "2026-10-27T02:58:43", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_trained_best_ufd.json b/repro/bench_results/npr_trained_best_ufd.json new file mode 100644 index 0000000000000000000000000000000000000000..f58a58b7ce351dddab53b2965bc0898f2696665e --- /dev/null +++ b/repro/bench_results/npr_trained_best_ufd.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_trained_best_ufd", + "model": "npr", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_best.pth", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 85.34, + "map": 95.56, + "mauroc": 95.3 + }, + "per_generator": { + "dalle": { + "ap": 92.85, + "auroc": 92.7, + "real_acc": 98.5, + "fake_acc": 44.5, + "acc": 71.5 + }, + "glide_100_10": { + "ap": 98.54, + "auroc": 98.22, + "real_acc": 98.5, + "fake_acc": 90.7, + "acc": 94.6 + }, + "glide_100_27": { + "ap": 97.85, + "auroc": 97.35, + "real_acc": 98.5, + "fake_acc": 86.0, + "acc": 92.25 + }, + "glide_50_27": { + "ap": 98.08, + "auroc": 97.6, + "real_acc": 98.5, + "fake_acc": 88.6, + "acc": 93.55 + }, + "guided": { + "ap": 84.94, + "auroc": 83.39, + "real_acc": 89.7, + "fake_acc": 58.6, + "acc": 74.15 + }, + "ldm_100": { + "ap": 97.13, + "auroc": 97.49, + "real_acc": 98.5, + "fake_acc": 69.3, + "acc": 83.9 + }, + "ldm_200": { + "ap": 97.01, + "auroc": 97.37, + "real_acc": 98.5, + "fake_acc": 67.6, + "acc": 83.05 + }, + "ldm_200_cfg": { + "ap": 98.09, + "auroc": 98.25, + "real_acc": 98.5, + "fake_acc": 80.9, + "acc": 89.7 + } + }, + "date": "2026-10-27T02:59:33", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_trained_last_forensynths8.json b/repro/bench_results/npr_trained_last_forensynths8.json new file mode 100644 index 0000000000000000000000000000000000000000..20b20320f74aaddce1a21c0fe67858a8f2f66804 --- /dev/null +++ b/repro/bench_results/npr_trained_last_forensynths8.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_trained_last_forensynths8", + "model": "npr", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_last.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 84.09, + "map": 93.83, + "mauroc": 94.94 + }, + "per_generator": { + "biggan": { + "ap": 91.45, + "auroc": 92.23, + "real_acc": 91.6, + "fake_acc": 68.9, + "acc": 80.25 + }, + "cyclegan": { + "ap": 99.07, + "auroc": 99.42, + "real_acc": 99.55, + "fake_acc": 47.84, + "acc": 73.69 + }, + "deepfake": { + "ap": 74.32, + "auroc": 78.49, + "real_acc": 90.47, + "fake_acc": 35.36, + "acc": 62.91 + }, + "gaugan": { + "ap": 86.04, + "auroc": 89.64, + "real_acc": 89.12, + "fake_acc": 59.62, + "acc": 74.37 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 100.0, + "fake_acc": 99.62, + "acc": 99.81 + }, + "stargan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 100.0, + "fake_acc": 80.49, + "acc": 90.25 + }, + "stylegan": { + "ap": 99.83, + "auroc": 99.8, + "real_acc": 100.0, + "fake_acc": 90.1, + "acc": 95.05 + }, + "stylegan2": { + "ap": 99.93, + "auroc": 99.92, + "real_acc": 100.0, + "fake_acc": 92.79, + "acc": 96.39 + } + }, + "date": "2026-10-27T02:53:54", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/npr_trained_last_ufd.json b/repro/bench_results/npr_trained_last_ufd.json new file mode 100644 index 0000000000000000000000000000000000000000..875a67bdb6736a6dddc649814f27dc9f3eea3660 --- /dev/null +++ b/repro/bench_results/npr_trained_last_ufd.json @@ -0,0 +1,74 @@ +{ + "tag": "npr_trained_last_ufd", + "model": "npr", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/npr_v1/model_last.pth", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 88.91, + "map": 97.49, + "mauroc": 97.23 + }, + "per_generator": { + "dalle": { + "ap": 95.29, + "auroc": 94.52, + "real_acc": 99.7, + "fake_acc": 38.1, + "acc": 68.9 + }, + "glide_100_10": { + "ap": 99.73, + "auroc": 99.73, + "real_acc": 99.7, + "fake_acc": 94.1, + "acc": 96.9 + }, + "glide_100_27": { + "ap": 99.55, + "auroc": 99.5, + "real_acc": 99.7, + "fake_acc": 89.7, + "acc": 94.7 + }, + "glide_50_27": { + "ap": 99.64, + "auroc": 99.6, + "real_acc": 99.7, + "fake_acc": 92.3, + "acc": 96.0 + }, + "guided": { + "ap": 86.62, + "auroc": 85.23, + "real_acc": 90.0, + "fake_acc": 59.9, + "acc": 74.95 + }, + "ldm_100": { + "ap": 99.71, + "auroc": 99.79, + "real_acc": 99.7, + "fake_acc": 87.0, + "acc": 93.35 + }, + "ldm_200": { + "ap": 99.7, + "auroc": 99.78, + "real_acc": 99.7, + "fake_acc": 85.3, + "acc": 92.5 + }, + "ldm_200_cfg": { + "ap": 99.7, + "auroc": 99.71, + "real_acc": 99.7, + "fake_acc": 88.3, + "acc": 94.0 + } + }, + "date": "2026-10-27T02:54:44", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_bfree_online.json b/repro/bench_results/omniaid_bfree_online.json new file mode 100644 index 0000000000000000000000000000000000000000..8b54c66f57357bf009f3780b0edc442f7cd8d9b3 --- /dev/null +++ b/repro/bench_results/omniaid_bfree_online.json @@ -0,0 +1,25 @@ +{ + "tag": "omniaid_bfree_online", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/BFreeOnline/test", + "jpeg_fakes": null, + "n_generators": 1, + "means": { + "acc": 70.26, + "map": 91.7, + "mauroc": 87.18 + }, + "per_generator": { + "viral": { + "ap": 91.7, + "auroc": 87.18, + "real_acc": 94.8, + "fake_acc": 45.72, + "acc": 70.26 + } + }, + "date": "2026-10-27T08:44:32", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_evalgen.json b/repro/bench_results/omniaid_evalgen.json new file mode 100644 index 0000000000000000000000000000000000000000..295c363aab1e52d7e08e33c7f5e4f80ad5e4c54d --- /dev/null +++ b/repro/bench_results/omniaid_evalgen.json @@ -0,0 +1,53 @@ +{ + "tag": "omniaid_evalgen", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/EvalGEN/test", + "jpeg_fakes": null, + "n_generators": 5, + "means": { + "acc": 74.14, + "map": 96.46, + "mauroc": 93.08 + }, + "per_generator": { + "Flux": { + "ap": 92.93, + "auroc": 86.17, + "real_acc": 98.22, + "fake_acc": 31.25, + "acc": 64.73 + }, + "GoT": { + "ap": 97.59, + "auroc": 95.55, + "real_acc": 98.22, + "fake_acc": 55.27, + "acc": 76.75 + }, + "Infinity": { + "ap": 97.4, + "auroc": 94.75, + "real_acc": 98.22, + "fake_acc": 56.64, + "acc": 77.43 + }, + "NOVA": { + "ap": 98.05, + "auroc": 96.14, + "real_acc": 98.22, + "fake_acc": 60.13, + "acc": 79.18 + }, + "OmniGen": { + "ap": 96.32, + "auroc": 92.77, + "real_acc": 98.22, + "fake_acc": 46.99, + "acc": 72.61 + } + }, + "date": "2026-10-27T11:05:31", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_genimage_aigcdetect.json b/repro/bench_results/omniaid_genimage_aigcdetect.json new file mode 100644 index 0000000000000000000000000000000000000000..a958c3229bc7c9667ad77446b6bd260e9a5b53be --- /dev/null +++ b/repro/bench_results/omniaid_genimage_aigcdetect.json @@ -0,0 +1,137 @@ +{ + "tag": "omniaid_genimage_aigcdetect", + "model": "omniaid_genimage", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/AIGCDetect/test", + "jpeg_fakes": null, + "n_generators": 17, + "means": { + "acc": 88.87, + "map": 97.88, + "mauroc": 98.2 + }, + "per_generator": { + "ADM": { + "ap": 97.72, + "auroc": 97.61, + "real_acc": 97.42, + "fake_acc": 85.32, + "acc": 91.37 + }, + "DALLE2": { + "ap": 99.4, + "auroc": 99.45, + "real_acc": 95.0, + "fake_acc": 98.9, + "acc": 96.95 + }, + "Glide": { + "ap": 99.74, + "auroc": 99.8, + "real_acc": 97.57, + "fake_acc": 99.77, + "acc": 98.67 + }, + "Midjourney": { + "ap": 95.42, + "auroc": 95.68, + "real_acc": 97.25, + "fake_acc": 74.12, + "acc": 85.68 + }, + "VQDM": { + "ap": 99.58, + "auroc": 99.59, + "real_acc": 97.65, + "fake_acc": 96.9, + "acc": 97.27 + }, + "biggan": { + "ap": 96.98, + "auroc": 97.8, + "real_acc": 73.65, + "fake_acc": 100.0, + "acc": 86.83 + }, + "cyclegan": { + "ap": 99.3, + "auroc": 99.48, + "real_acc": 85.24, + "fake_acc": 100.0, + "acc": 92.62 + }, + "gaugan": { + "ap": 96.45, + "auroc": 97.94, + "real_acc": 72.94, + "fake_acc": 100.0, + "acc": 86.47 + }, + "progan": { + "ap": 96.34, + "auroc": 96.77, + "real_acc": 13.22, + "fake_acc": 100.0, + "acc": 56.61 + }, + "sd_xl": { + "ap": 98.25, + "auroc": 98.65, + "real_acc": 97.4, + "fake_acc": 91.1, + "acc": 94.25 + }, + "stable_diffusion_v_1_4": { + "ap": 99.9, + "auroc": 99.92, + "real_acc": 97.9, + "fake_acc": 99.93, + "acc": 98.92 + }, + "stable_diffusion_v_1_5": { + "ap": 99.81, + "auroc": 99.86, + "real_acc": 97.62, + "fake_acc": 99.94, + "acc": 98.78 + }, + "stargan": { + "ap": 99.03, + "auroc": 99.25, + "real_acc": 76.89, + "fake_acc": 100.0, + "acc": 88.44 + }, + "stylegan": { + "ap": 95.73, + "auroc": 96.17, + "real_acc": 58.97, + "fake_acc": 98.88, + "acc": 78.93 + }, + "stylegan2": { + "ap": 92.52, + "auroc": 93.05, + "real_acc": 51.09, + "fake_acc": 98.22, + "acc": 74.66 + }, + "whichfaceisreal": { + "ap": 98.05, + "auroc": 98.66, + "real_acc": 72.4, + "fake_acc": 100.0, + "acc": 86.2 + }, + "wukong": { + "ap": 99.78, + "auroc": 99.79, + "real_acc": 97.67, + "fake_acc": 98.47, + "acc": 98.07 + } + }, + "date": "2026-10-27T13:04:46", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_genimage_genimage_full.json b/repro/bench_results/omniaid_genimage_genimage_full.json new file mode 100644 index 0000000000000000000000000000000000000000..5c1893f93c4b08069805b01f415653967176a062 --- /dev/null +++ b/repro/bench_results/omniaid_genimage_genimage_full.json @@ -0,0 +1,74 @@ +{ + "tag": "omniaid_genimage_genimage_full", + "model": "omniaid_genimage", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 95.82, + "map": 99.02, + "mauroc": 99.16 + }, + "per_generator": { + "adm": { + "ap": 97.91, + "auroc": 97.83, + "real_acc": 98.0, + "fake_acc": 82.8, + "acc": 90.4 + }, + "biggan": { + "ap": 99.86, + "auroc": 99.9, + "real_acc": 97.67, + "fake_acc": 99.73, + "acc": 98.7 + }, + "glide": { + "ap": 99.52, + "auroc": 99.73, + "real_acc": 97.93, + "fake_acc": 100.0, + "acc": 98.97 + }, + "midjourney": { + "ap": 96.4, + "auroc": 97.17, + "real_acc": 97.8, + "fake_acc": 76.6, + "acc": 87.2 + }, + "sdv1.4": { + "ap": 99.9, + "auroc": 99.92, + "real_acc": 97.9, + "fake_acc": 99.93, + "acc": 98.92 + }, + "sdv1.5": { + "ap": 99.66, + "auroc": 99.75, + "real_acc": 97.6, + "fake_acc": 99.93, + "acc": 98.77 + }, + "vqdm": { + "ap": 99.23, + "auroc": 99.32, + "real_acc": 98.07, + "fake_acc": 94.07, + "acc": 96.07 + }, + "wukong": { + "ap": 99.65, + "auroc": 99.68, + "real_acc": 97.73, + "fake_acc": 97.27, + "acc": 97.5 + } + }, + "date": "2026-10-27T09:20:42", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_genimage_mirage_test.json b/repro/bench_results/omniaid_genimage_mirage_test.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d54284e1d8137df12c11dd336f54411cd1ca1b --- /dev/null +++ b/repro/bench_results/omniaid_genimage_mirage_test.json @@ -0,0 +1,53 @@ +{ + "tag": "omniaid_genimage_mirage_test", + "model": "omniaid_genimage", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/MirageTest/test", + "jpeg_fakes": null, + "n_generators": 5, + "means": { + "acc": 54.66, + "map": 50.6, + "mauroc": 50.74 + }, + "per_generator": { + "Animal": { + "ap": 59.07, + "auroc": 53.39, + "real_acc": 82.63, + "fake_acc": 29.17, + "acc": 55.9 + }, + "Anime": { + "ap": 39.42, + "auroc": 65.51, + "real_acc": 80.32, + "fake_acc": 44.67, + "acc": 62.49 + }, + "Human": { + "ap": 64.73, + "auroc": 59.25, + "real_acc": 76.4, + "fake_acc": 42.35, + "acc": 59.38 + }, + "Object": { + "ap": 46.57, + "auroc": 40.8, + "real_acc": 82.6, + "fake_acc": 15.43, + "acc": 49.02 + }, + "Scene": { + "ap": 43.2, + "auroc": 34.75, + "real_acc": 80.6, + "fake_acc": 12.45, + "acc": 46.52 + } + }, + "date": "2026-10-27T15:40:59", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_mirage_test.json b/repro/bench_results/omniaid_mirage_test.json new file mode 100644 index 0000000000000000000000000000000000000000..e332cc7c2f2c59d4221c108d62682862bc4c5e75 --- /dev/null +++ b/repro/bench_results/omniaid_mirage_test.json @@ -0,0 +1,53 @@ +{ + "tag": "omniaid_mirage_test", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/MirageTest/test", + "jpeg_fakes": null, + "n_generators": 5, + "means": { + "acc": 80.37, + "map": 96.48, + "mauroc": 96.65 + }, + "per_generator": { + "Animal": { + "ap": 95.65, + "auroc": 95.51, + "real_acc": 98.86, + "fake_acc": 55.54, + "acc": 77.2 + }, + "Anime": { + "ap": 97.03, + "auroc": 98.86, + "real_acc": 99.41, + "fake_acc": 80.28, + "acc": 89.85 + }, + "Human": { + "ap": 99.15, + "auroc": 99.08, + "real_acc": 99.67, + "fake_acc": 77.98, + "acc": 88.82 + }, + "Object": { + "ap": 95.54, + "auroc": 95.33, + "real_acc": 99.07, + "fake_acc": 51.12, + "acc": 75.09 + }, + "Scene": { + "ap": 95.05, + "auroc": 94.49, + "real_acc": 99.82, + "fake_acc": 41.97, + "acc": 70.89 + } + }, + "date": "2026-10-27T16:20:10", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_realchain.json b/repro/bench_results/omniaid_realchain.json new file mode 100644 index 0000000000000000000000000000000000000000..a407d12051f6ea253a9ce1589ad52c915c031f1f --- /dev/null +++ b/repro/bench_results/omniaid_realchain.json @@ -0,0 +1,67 @@ +{ + "tag": "omniaid_realchain", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/RealChain/test", + "jpeg_fakes": null, + "n_generators": 7, + "means": { + "acc": 61.71, + "map": 84.02, + "mauroc": 83.63 + }, + "per_generator": { + "Flux1": { + "ap": 85.22, + "auroc": 84.23, + "real_acc": 97.2, + "fake_acc": 36.9, + "acc": 67.05 + }, + "Hunyuan3": { + "ap": 92.41, + "auroc": 91.87, + "real_acc": 98.3, + "fake_acc": 49.1, + "acc": 73.7 + }, + "NanoBanana": { + "ap": 92.88, + "auroc": 92.71, + "real_acc": 99.7, + "fake_acc": 29.9, + "acc": 64.8 + }, + "QwenImage": { + "ap": 81.66, + "auroc": 80.19, + "real_acc": 98.7, + "fake_acc": 21.4, + "acc": 60.05 + }, + "SDv3.5": { + "ap": 72.86, + "auroc": 72.64, + "real_acc": 99.0, + "fake_acc": 11.6, + "acc": 55.3 + }, + "Seedream4": { + "ap": 85.0, + "auroc": 85.91, + "real_acc": 98.8, + "fake_acc": 15.3, + "acc": 57.05 + }, + "i2i": { + "ap": 78.1, + "auroc": 77.83, + "real_acc": 99.3, + "fake_acc": 8.7, + "acc": 54.0 + } + }, + "date": "2026-10-27T09:09:34", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_rrdataset.json b/repro/bench_results/omniaid_rrdataset.json new file mode 100644 index 0000000000000000000000000000000000000000..0f45972af93130811bb0836309b5f9d4f66e316b --- /dev/null +++ b/repro/bench_results/omniaid_rrdataset.json @@ -0,0 +1,39 @@ +{ + "tag": "omniaid_rrdataset", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/RRDataset/test", + "jpeg_fakes": null, + "n_generators": 3, + "means": { + "acc": 71.75, + "map": 84.19, + "mauroc": 82.32 + }, + "per_generator": { + "original": { + "ap": 96.75, + "auroc": 96.2, + "real_acc": 95.55, + "fake_acc": 83.86, + "acc": 89.71 + }, + "redigital": { + "ap": 77.4, + "auroc": 74.41, + "real_acc": 91.03, + "fake_acc": 41.78, + "acc": 66.41 + }, + "transfer": { + "ap": 78.43, + "auroc": 76.36, + "real_acc": 98.88, + "fake_acc": 19.4, + "acc": 59.14 + } + }, + "date": "2026-10-27T12:23:42", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_synthwildx.json b/repro/bench_results/omniaid_synthwildx.json new file mode 100644 index 0000000000000000000000000000000000000000..866f9aa7a389da4a5b810fb84ae39508a7d52fe6 --- /dev/null +++ b/repro/bench_results/omniaid_synthwildx.json @@ -0,0 +1,39 @@ +{ + "tag": "omniaid_synthwildx", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/SynthWildX/test", + "jpeg_fakes": null, + "n_generators": 3, + "means": { + "acc": 86.02, + "map": 95.52, + "mauroc": 95.31 + }, + "per_generator": { + "dalle3": { + "ap": 98.74, + "auroc": 98.93, + "real_acc": 95.88, + "fake_acc": 93.88, + "acc": 94.88 + }, + "firefly": { + "ap": 91.41, + "auroc": 90.46, + "real_acc": 95.88, + "fake_acc": 56.45, + "acc": 76.17 + }, + "midjourney_v5": { + "ap": 96.42, + "auroc": 96.55, + "real_acc": 95.88, + "fake_acc": 78.16, + "acc": 87.02 + } + }, + "date": "2026-10-27T08:37:18", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_v1_aigcdetect.json b/repro/bench_results/omniaid_v1_aigcdetect.json new file mode 100644 index 0000000000000000000000000000000000000000..208be393c540fc715825645d526d898fdbf9dc61 --- /dev/null +++ b/repro/bench_results/omniaid_v1_aigcdetect.json @@ -0,0 +1,137 @@ +{ + "tag": "omniaid_v1_aigcdetect", + "model": "omniaid_v1", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/AIGCDetect/test", + "jpeg_fakes": null, + "n_generators": 17, + "means": { + "acc": 92.88, + "map": 98.93, + "mauroc": 98.99 + }, + "per_generator": { + "ADM": { + "ap": 96.32, + "auroc": 95.43, + "real_acc": 96.92, + "fake_acc": 82.08, + "acc": 89.5 + }, + "DALLE2": { + "ap": 99.88, + "auroc": 99.9, + "real_acc": 96.4, + "fake_acc": 100.0, + "acc": 98.2 + }, + "Glide": { + "ap": 99.49, + "auroc": 99.7, + "real_acc": 97.28, + "fake_acc": 99.4, + "acc": 98.34 + }, + "Midjourney": { + "ap": 99.65, + "auroc": 99.68, + "real_acc": 97.32, + "fake_acc": 98.7, + "acc": 98.01 + }, + "VQDM": { + "ap": 99.83, + "auroc": 99.85, + "real_acc": 97.05, + "fake_acc": 99.7, + "acc": 98.37 + }, + "biggan": { + "ap": 94.74, + "auroc": 96.19, + "real_acc": 66.45, + "fake_acc": 99.15, + "acc": 82.8 + }, + "cyclegan": { + "ap": 98.27, + "auroc": 98.46, + "real_acc": 80.92, + "fake_acc": 99.62, + "acc": 90.27 + }, + "gaugan": { + "ap": 98.72, + "auroc": 98.64, + "real_acc": 67.66, + "fake_acc": 99.5, + "acc": 83.58 + }, + "progan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 61.8, + "fake_acc": 100.0, + "acc": 80.9 + }, + "sd_xl": { + "ap": 99.94, + "auroc": 99.94, + "real_acc": 97.8, + "fake_acc": 99.9, + "acc": 98.85 + }, + "stable_diffusion_v_1_4": { + "ap": 99.94, + "auroc": 99.96, + "real_acc": 97.38, + "fake_acc": 100.0, + "acc": 98.69 + }, + "stable_diffusion_v_1_5": { + "ap": 99.95, + "auroc": 99.96, + "real_acc": 96.84, + "fake_acc": 99.87, + "acc": 98.36 + }, + "stargan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 97.8, + "fake_acc": 100.0, + "acc": 98.9 + }, + "stylegan": { + "ap": 99.44, + "auroc": 99.36, + "real_acc": 82.64, + "fake_acc": 98.9, + "acc": 90.77 + }, + "stylegan2": { + "ap": 99.04, + "auroc": 98.95, + "real_acc": 74.67, + "fake_acc": 98.95, + "acc": 86.81 + }, + "whichfaceisreal": { + "ap": 96.68, + "auroc": 96.78, + "real_acc": 80.6, + "fake_acc": 95.5, + "acc": 88.05 + }, + "wukong": { + "ap": 99.95, + "auroc": 99.96, + "real_acc": 97.33, + "fake_acc": 99.87, + "acc": 98.6 + } + }, + "date": "2026-10-27T13:11:33", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_v1_genimage_full.json b/repro/bench_results/omniaid_v1_genimage_full.json new file mode 100644 index 0000000000000000000000000000000000000000..a21fe42fc8ed95864e4359844fb58252e6d9b95d --- /dev/null +++ b/repro/bench_results/omniaid_v1_genimage_full.json @@ -0,0 +1,74 @@ +{ + "tag": "omniaid_v1_genimage_full", + "model": "omniaid_v1", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/GenImage_full/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 97.16, + "map": 99.2, + "mauroc": 99.12 + }, + "per_generator": { + "adm": { + "ap": 95.77, + "auroc": 94.37, + "real_acc": 97.4, + "fake_acc": 80.4, + "acc": 88.9 + }, + "biggan": { + "ap": 99.43, + "auroc": 99.61, + "real_acc": 97.07, + "fake_acc": 98.67, + "acc": 97.87 + }, + "glide": { + "ap": 99.19, + "auroc": 99.66, + "real_acc": 96.73, + "fake_acc": 99.67, + "acc": 98.2 + }, + "midjourney": { + "ap": 99.66, + "auroc": 99.7, + "real_acc": 97.2, + "fake_acc": 99.2, + "acc": 98.2 + }, + "sdv1.4": { + "ap": 99.94, + "auroc": 99.96, + "real_acc": 97.38, + "fake_acc": 100.0, + "acc": 98.69 + }, + "sdv1.5": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 97.27, + "fake_acc": 100.0, + "acc": 98.63 + }, + "vqdm": { + "ap": 99.67, + "auroc": 99.76, + "real_acc": 97.27, + "fake_acc": 99.53, + "acc": 98.4 + }, + "wukong": { + "ap": 99.95, + "auroc": 99.95, + "real_acc": 97.07, + "fake_acc": 99.8, + "acc": 98.43 + } + }, + "date": "2026-10-27T09:22:13", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_v1_mirage_test.json b/repro/bench_results/omniaid_v1_mirage_test.json new file mode 100644 index 0000000000000000000000000000000000000000..a6b601b59c1e6fe2558b277b659a3dddfebcfd00 --- /dev/null +++ b/repro/bench_results/omniaid_v1_mirage_test.json @@ -0,0 +1,53 @@ +{ + "tag": "omniaid_v1_mirage_test", + "model": "omniaid_v1", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/MirageTest/test", + "jpeg_fakes": null, + "n_generators": 5, + "means": { + "acc": 88.63, + "map": 97.04, + "mauroc": 97.97 + }, + "per_generator": { + "Animal": { + "ap": 92.57, + "auroc": 94.81, + "real_acc": 93.69, + "fake_acc": 69.06, + "acc": 81.37 + }, + "Anime": { + "ap": 97.96, + "auroc": 99.43, + "real_acc": 97.25, + "fake_acc": 94.92, + "acc": 96.08 + }, + "Human": { + "ap": 99.18, + "auroc": 99.33, + "real_acc": 98.13, + "fake_acc": 89.25, + "acc": 93.69 + }, + "Object": { + "ap": 97.02, + "auroc": 97.7, + "real_acc": 97.17, + "fake_acc": 72.67, + "acc": 84.92 + }, + "Scene": { + "ap": 98.47, + "auroc": 98.6, + "real_acc": 98.53, + "fake_acc": 75.6, + "acc": 87.07 + } + }, + "date": "2026-10-27T15:49:26", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/omniaid_wildrf.json b/repro/bench_results/omniaid_wildrf.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbba5992003542d25d54e54da4603c9e21e92bd --- /dev/null +++ b/repro/bench_results/omniaid_wildrf.json @@ -0,0 +1,39 @@ +{ + "tag": "omniaid_wildrf", + "model": "omniaid", + "ckpt": "registry-default", + "benchmark": "pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/WildRF/test", + "jpeg_fakes": null, + "n_generators": 3, + "means": { + "acc": 88.36, + "map": 97.28, + "mauroc": 96.91 + }, + "per_generator": { + "facebook": { + "ap": 97.39, + "auroc": 96.75, + "real_acc": 97.5, + "fake_acc": 82.5, + "acc": 90.0 + }, + "reddit": { + "ap": 96.67, + "auroc": 96.14, + "real_acc": 97.33, + "fake_acc": 76.8, + "acc": 87.07 + }, + "twitter": { + "ap": 97.79, + "auroc": 97.84, + "real_acc": 96.77, + "fake_acc": 79.24, + "acc": 88.01 + } + }, + "date": "2026-10-27T08:42:46", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_official_forensynths13.json b/repro/bench_results/pgc_official_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..6048f59e1433b542c16887b1c44e8ddfccc8cf2e --- /dev/null +++ b/repro/bench_results/pgc_official_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_official_forensynths13", + "model": "pgc", + "ckpt": "registry-default", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 70.58, + "map": 93.8, + "mauroc": 94.37 + }, + "per_generator": { + "biggan": { + "ap": 97.37, + "auroc": 97.35, + "real_acc": 98.8, + "fake_acc": 67.05, + "acc": 82.92 + }, + "crn": { + "ap": 95.59, + "auroc": 96.27, + "real_acc": 99.78, + "fake_acc": 19.23, + "acc": 59.5 + }, + "cyclegan": { + "ap": 99.68, + "auroc": 99.74, + "real_acc": 99.7, + "fake_acc": 83.42, + "acc": 91.56 + }, + "deepfake": { + "ap": 92.5, + "auroc": 96.03, + "real_acc": 98.6, + "fake_acc": 12.64, + "acc": 55.62 + }, + "gaugan": { + "ap": 99.46, + "auroc": 99.45, + "real_acc": 99.66, + "fake_acc": 86.56, + "acc": 93.11 + }, + "imle": { + "ap": 98.79, + "auroc": 98.94, + "real_acc": 99.78, + "fake_acc": 55.78, + "acc": 77.78 + }, + "progan": { + "ap": 99.18, + "auroc": 99.06, + "real_acc": 99.15, + "fake_acc": 87.65, + "acc": 93.4 + }, + "san": { + "ap": 98.35, + "auroc": 98.67, + "real_acc": 100.0, + "fake_acc": 29.22, + "acc": 64.61 + }, + "seeingdark": { + "ap": 79.53, + "auroc": 77.9, + "real_acc": 10.56, + "fake_acc": 99.44, + "acc": 55.0 + }, + "stargan": { + "ap": 97.45, + "auroc": 97.71, + "real_acc": 98.7, + "fake_acc": 61.68, + "acc": 80.19 + }, + "stylegan": { + "ap": 93.49, + "auroc": 94.62, + "real_acc": 99.8, + "fake_acc": 15.86, + "acc": 57.83 + }, + "stylegan2": { + "ap": 92.11, + "auroc": 92.54, + "real_acc": 99.81, + "fake_acc": 12.34, + "acc": 56.08 + }, + "whichfaceisreal": { + "ap": 75.92, + "auroc": 78.55, + "real_acc": 100.0, + "fake_acc": 0.0, + "acc": 50.0 + } + }, + "date": "2026-10-27T18:34:34", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_official_ufd8.json b/repro/bench_results/pgc_official_ufd8.json new file mode 100644 index 0000000000000000000000000000000000000000..5b182774b4e1f850d424e21dfc471ec10ed556bd --- /dev/null +++ b/repro/bench_results/pgc_official_ufd8.json @@ -0,0 +1,74 @@ +{ + "tag": "pgc_official_ufd8", + "model": "pgc", + "ckpt": "registry-default", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 83.57, + "map": 99.17, + "mauroc": 99.2 + }, + "per_generator": { + "dalle": { + "ap": 98.76, + "auroc": 98.88, + "real_acc": 99.8, + "fake_acc": 59.7, + "acc": 79.75 + }, + "glide_100_10": { + "ap": 99.13, + "auroc": 99.13, + "real_acc": 99.8, + "fake_acc": 73.6, + "acc": 86.7 + }, + "glide_100_27": { + "ap": 99.14, + "auroc": 99.16, + "real_acc": 99.8, + "fake_acc": 72.3, + "acc": 86.05 + }, + "glide_50_27": { + "ap": 99.05, + "auroc": 99.06, + "real_acc": 99.8, + "fake_acc": 71.5, + "acc": 85.65 + }, + "guided": { + "ap": 99.75, + "auroc": 99.77, + "real_acc": 100.0, + "fake_acc": 29.7, + "acc": 64.85 + }, + "ldm_100": { + "ap": 99.57, + "auroc": 99.55, + "real_acc": 99.8, + "fake_acc": 86.1, + "acc": 92.95 + }, + "ldm_200": { + "ap": 99.67, + "auroc": 99.66, + "real_acc": 99.8, + "fake_acc": 86.1, + "acc": 92.95 + }, + "ldm_200_cfg": { + "ap": 98.3, + "auroc": 98.37, + "real_acc": 99.8, + "fake_acc": 59.5, + "acc": 79.65 + } + }, + "date": "2026-10-28T00:46:03", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step1100_forensynths13.json b/repro/bench_results/pgc_trained_step1100_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..7bf424863dcfbf50f7ef6ec7a3e26801638a4b3c --- /dev/null +++ b/repro/bench_results/pgc_trained_step1100_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step1100_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1100.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 75.44, + "map": 96.74, + "mauroc": 96.5 + }, + "per_generator": { + "biggan": { + "ap": 99.78, + "auroc": 99.72, + "real_acc": 99.85, + "fake_acc": 94.2, + "acc": 97.02 + }, + "crn": { + "ap": 97.57, + "auroc": 97.83, + "real_acc": 99.97, + "fake_acc": 19.77, + "acc": 59.87 + }, + "cyclegan": { + "ap": 99.88, + "auroc": 99.87, + "real_acc": 99.92, + "fake_acc": 92.05, + "acc": 95.99 + }, + "deepfake": { + "ap": 90.92, + "auroc": 89.37, + "real_acc": 100.0, + "fake_acc": 3.0, + "acc": 51.5 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.96, + "fake_acc": 99.48, + "acc": 99.72 + }, + "imle": { + "ap": 99.18, + "auroc": 99.21, + "real_acc": 99.97, + "fake_acc": 48.89, + "acc": 74.43 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.92, + "fake_acc": 99.77, + "acc": 99.85 + }, + "san": { + "ap": 99.05, + "auroc": 99.13, + "real_acc": 100.0, + "fake_acc": 34.25, + "acc": 67.12 + }, + "seeingdark": { + "ap": 89.7, + "auroc": 89.5, + "real_acc": 100.0, + "fake_acc": 18.89, + "acc": 59.44 + }, + "stargan": { + "ap": 97.68, + "auroc": 97.76, + "real_acc": 99.85, + "fake_acc": 50.13, + "acc": 74.99 + }, + "stylegan": { + "ap": 95.71, + "auroc": 95.23, + "real_acc": 99.9, + "fake_acc": 58.67, + "acc": 79.29 + }, + "stylegan2": { + "ap": 94.86, + "auroc": 94.01, + "real_acc": 99.9, + "fake_acc": 42.94, + "acc": 71.42 + }, + "whichfaceisreal": { + "ap": 93.36, + "auroc": 92.86, + "real_acc": 100.0, + "fake_acc": 0.1, + "acc": 50.05 + } + }, + "date": "2026-10-27T21:15:31", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step1300_forensynths13.json b/repro/bench_results/pgc_trained_step1300_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..6da50c387d9a4c8a3987c4fd18de5fccca12ee7e --- /dev/null +++ b/repro/bench_results/pgc_trained_step1300_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step1300_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1300.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 78.24, + "map": 97.03, + "mauroc": 96.8 + }, + "per_generator": { + "biggan": { + "ap": 99.88, + "auroc": 99.86, + "real_acc": 99.85, + "fake_acc": 96.3, + "acc": 98.07 + }, + "crn": { + "ap": 97.77, + "auroc": 97.99, + "real_acc": 99.83, + "fake_acc": 34.79, + "acc": 67.31 + }, + "cyclegan": { + "ap": 99.94, + "auroc": 99.93, + "real_acc": 99.77, + "fake_acc": 95.61, + "acc": 97.69 + }, + "deepfake": { + "ap": 91.25, + "auroc": 89.59, + "real_acc": 100.0, + "fake_acc": 4.15, + "acc": 52.08 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.96, + "fake_acc": 99.76, + "acc": 99.86 + }, + "imle": { + "ap": 99.26, + "auroc": 99.27, + "real_acc": 99.83, + "fake_acc": 67.42, + "acc": 83.63 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.83, + "fake_acc": 99.95, + "acc": 99.89 + }, + "san": { + "ap": 99.09, + "auroc": 99.17, + "real_acc": 100.0, + "fake_acc": 42.92, + "acc": 71.46 + }, + "seeingdark": { + "ap": 89.56, + "auroc": 89.53, + "real_acc": 100.0, + "fake_acc": 18.89, + "acc": 59.44 + }, + "stargan": { + "ap": 97.96, + "auroc": 98.02, + "real_acc": 99.7, + "fake_acc": 58.63, + "acc": 79.16 + }, + "stylegan": { + "ap": 96.01, + "auroc": 95.52, + "real_acc": 99.83, + "fake_acc": 64.66, + "acc": 82.25 + }, + "stylegan2": { + "ap": 96.02, + "auroc": 95.32, + "real_acc": 99.82, + "fake_acc": 51.93, + "acc": 75.88 + }, + "whichfaceisreal": { + "ap": 94.66, + "auroc": 94.24, + "real_acc": 100.0, + "fake_acc": 0.9, + "acc": 50.45 + } + }, + "date": "2026-10-27T21:47:29", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step1500_forensynths13.json b/repro/bench_results/pgc_trained_step1500_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..9f7fc4a9291067f9f70dff203b91098b38b46ca6 --- /dev/null +++ b/repro/bench_results/pgc_trained_step1500_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step1500_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1500.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 77.2, + "map": 96.85, + "mauroc": 96.54 + }, + "per_generator": { + "biggan": { + "ap": 99.86, + "auroc": 99.82, + "real_acc": 99.85, + "fake_acc": 96.15, + "acc": 98.0 + }, + "crn": { + "ap": 97.72, + "auroc": 97.93, + "real_acc": 99.97, + "fake_acc": 27.15, + "acc": 63.56 + }, + "cyclegan": { + "ap": 99.9, + "auroc": 99.89, + "real_acc": 99.85, + "fake_acc": 93.94, + "acc": 96.9 + }, + "deepfake": { + "ap": 89.72, + "auroc": 87.55, + "real_acc": 100.0, + "fake_acc": 3.82, + "acc": 51.91 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.96, + "fake_acc": 99.66, + "acc": 99.81 + }, + "imle": { + "ap": 99.22, + "auroc": 99.24, + "real_acc": 99.97, + "fake_acc": 59.26, + "acc": 79.61 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.95, + "fake_acc": 99.98, + "acc": 99.96 + }, + "san": { + "ap": 98.87, + "auroc": 98.94, + "real_acc": 100.0, + "fake_acc": 40.18, + "acc": 70.09 + }, + "seeingdark": { + "ap": 89.34, + "auroc": 89.13, + "real_acc": 100.0, + "fake_acc": 18.33, + "acc": 59.17 + }, + "stargan": { + "ap": 97.81, + "auroc": 97.87, + "real_acc": 99.8, + "fake_acc": 54.28, + "acc": 77.04 + }, + "stylegan": { + "ap": 96.0, + "auroc": 95.46, + "real_acc": 99.87, + "fake_acc": 64.35, + "acc": 82.11 + }, + "stylegan2": { + "ap": 95.49, + "auroc": 94.5, + "real_acc": 99.86, + "fake_acc": 49.24, + "acc": 74.55 + }, + "whichfaceisreal": { + "ap": 95.11, + "auroc": 94.69, + "real_acc": 100.0, + "fake_acc": 1.7, + "acc": 50.85 + } + }, + "date": "2026-10-27T22:19:20", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step1700_forensynths13.json b/repro/bench_results/pgc_trained_step1700_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..601fa04f1def8c96f41071034c0a97945504ce2b --- /dev/null +++ b/repro/bench_results/pgc_trained_step1700_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step1700_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1700.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 74.18, + "map": 96.69, + "mauroc": 96.32 + }, + "per_generator": { + "biggan": { + "ap": 99.83, + "auroc": 99.79, + "real_acc": 99.85, + "fake_acc": 95.5, + "acc": 97.67 + }, + "crn": { + "ap": 97.43, + "auroc": 97.63, + "real_acc": 100.0, + "fake_acc": 12.19, + "acc": 56.1 + }, + "cyclegan": { + "ap": 99.91, + "auroc": 99.9, + "real_acc": 99.92, + "fake_acc": 91.98, + "acc": 95.95 + }, + "deepfake": { + "ap": 88.42, + "auroc": 86.09, + "real_acc": 100.0, + "fake_acc": 2.26, + "acc": 51.13 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.98, + "fake_acc": 99.36, + "acc": 99.67 + }, + "imle": { + "ap": 99.22, + "auroc": 99.25, + "real_acc": 100.0, + "fake_acc": 36.84, + "acc": 68.42 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.95, + "fake_acc": 99.95, + "acc": 99.95 + }, + "san": { + "ap": 99.0, + "auroc": 99.1, + "real_acc": 100.0, + "fake_acc": 22.37, + "acc": 61.19 + }, + "seeingdark": { + "ap": 88.66, + "auroc": 87.9, + "real_acc": 100.0, + "fake_acc": 11.67, + "acc": 55.83 + }, + "stargan": { + "ap": 98.05, + "auroc": 98.15, + "real_acc": 99.95, + "fake_acc": 47.27, + "acc": 73.61 + }, + "stylegan": { + "ap": 96.41, + "auroc": 95.99, + "real_acc": 99.93, + "fake_acc": 63.03, + "acc": 81.48 + }, + "stylegan2": { + "ap": 95.08, + "auroc": 93.99, + "real_acc": 99.92, + "fake_acc": 46.14, + "acc": 73.03 + }, + "whichfaceisreal": { + "ap": 94.93, + "auroc": 94.37, + "real_acc": 100.0, + "fake_acc": 0.6, + "acc": 50.3 + } + }, + "date": "2026-10-27T22:51:19", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step1900_forensynths13.json b/repro/bench_results/pgc_trained_step1900_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..6f909f815839f5d7df015fa1e82e210bba14fe53 --- /dev/null +++ b/repro/bench_results/pgc_trained_step1900_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step1900_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step1900.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 74.49, + "map": 95.97, + "mauroc": 95.35 + }, + "per_generator": { + "biggan": { + "ap": 99.74, + "auroc": 99.62, + "real_acc": 99.9, + "fake_acc": 94.7, + "acc": 97.3 + }, + "crn": { + "ap": 97.89, + "auroc": 98.06, + "real_acc": 100.0, + "fake_acc": 17.13, + "acc": 58.56 + }, + "cyclegan": { + "ap": 99.82, + "auroc": 99.78, + "real_acc": 99.92, + "fake_acc": 91.45, + "acc": 95.69 + }, + "deepfake": { + "ap": 83.61, + "auroc": 79.98, + "real_acc": 100.0, + "fake_acc": 1.78, + "acc": 50.89 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.98, + "fake_acc": 99.3, + "acc": 99.64 + }, + "imle": { + "ap": 99.37, + "auroc": 99.39, + "real_acc": 100.0, + "fake_acc": 47.96, + "acc": 73.98 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.95, + "fake_acc": 99.98, + "acc": 99.96 + }, + "san": { + "ap": 98.09, + "auroc": 98.07, + "real_acc": 100.0, + "fake_acc": 24.2, + "acc": 62.1 + }, + "seeingdark": { + "ap": 88.55, + "auroc": 87.7, + "real_acc": 100.0, + "fake_acc": 9.44, + "acc": 54.72 + }, + "stargan": { + "ap": 97.21, + "auroc": 97.23, + "real_acc": 99.9, + "fake_acc": 43.17, + "acc": 71.54 + }, + "stylegan": { + "ap": 95.68, + "auroc": 94.89, + "real_acc": 99.95, + "fake_acc": 63.63, + "acc": 81.79 + }, + "stylegan2": { + "ap": 93.52, + "auroc": 91.49, + "real_acc": 99.94, + "fake_acc": 43.01, + "acc": 71.48 + }, + "whichfaceisreal": { + "ap": 94.12, + "auroc": 93.37, + "real_acc": 100.0, + "fake_acc": 1.4, + "acc": 50.7 + } + }, + "date": "2026-10-27T23:23:08", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step300_forensynths13.json b/repro/bench_results/pgc_trained_step300_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..129b3da80bceec14c6a2bc6aaea52944819029e9 --- /dev/null +++ b/repro/bench_results/pgc_trained_step300_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step300_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step300.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 79.88, + "map": 95.07, + "mauroc": 95.66 + }, + "per_generator": { + "biggan": { + "ap": 99.58, + "auroc": 99.53, + "real_acc": 83.85, + "fake_acc": 99.45, + "acc": 91.65 + }, + "crn": { + "ap": 96.52, + "auroc": 97.31, + "real_acc": 15.21, + "fake_acc": 100.0, + "acc": 57.61 + }, + "cyclegan": { + "ap": 99.92, + "auroc": 99.94, + "real_acc": 97.96, + "fake_acc": 100.0, + "acc": 98.98 + }, + "deepfake": { + "ap": 93.18, + "auroc": 96.37, + "real_acc": 94.13, + "fake_acc": 90.18, + "acc": 92.15 + }, + "gaugan": { + "ap": 99.98, + "auroc": 99.98, + "real_acc": 86.58, + "fake_acc": 100.0, + "acc": 93.29 + }, + "imle": { + "ap": 99.22, + "auroc": 99.41, + "real_acc": 15.21, + "fake_acc": 100.0, + "acc": 57.61 + }, + "progan": { + "ap": 99.65, + "auroc": 99.53, + "real_acc": 85.18, + "fake_acc": 99.25, + "acc": 92.21 + }, + "san": { + "ap": 99.15, + "auroc": 99.27, + "real_acc": 96.8, + "fake_acc": 95.89, + "acc": 96.35 + }, + "seeingdark": { + "ap": 82.32, + "auroc": 83.01, + "real_acc": 0.56, + "fake_acc": 100.0, + "acc": 50.28 + }, + "stargan": { + "ap": 98.2, + "auroc": 98.25, + "real_acc": 74.79, + "fake_acc": 99.7, + "acc": 87.24 + }, + "stylegan": { + "ap": 93.91, + "auroc": 93.54, + "real_acc": 94.42, + "fake_acc": 71.12, + "acc": 82.77 + }, + "stylegan2": { + "ap": 96.36, + "auroc": 96.39, + "real_acc": 94.45, + "fake_acc": 82.09, + "acc": 88.27 + }, + "whichfaceisreal": { + "ap": 77.94, + "auroc": 81.02, + "real_acc": 100.0, + "fake_acc": 0.0, + "acc": 50.0 + } + }, + "date": "2026-10-27T19:07:32", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step500_forensynths13.json b/repro/bench_results/pgc_trained_step500_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..592ee7b5e17886e70a19cd2d5db7c09268e86677 --- /dev/null +++ b/repro/bench_results/pgc_trained_step500_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step500_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step500.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 82.98, + "map": 95.78, + "mauroc": 96.15 + }, + "per_generator": { + "biggan": { + "ap": 99.57, + "auroc": 99.52, + "real_acc": 99.15, + "fake_acc": 94.7, + "acc": 96.93 + }, + "crn": { + "ap": 96.92, + "auroc": 97.39, + "real_acc": 92.78, + "fake_acc": 90.33, + "acc": 91.55 + }, + "cyclegan": { + "ap": 99.9, + "auroc": 99.9, + "real_acc": 99.77, + "fake_acc": 95.76, + "acc": 97.77 + }, + "deepfake": { + "ap": 93.02, + "auroc": 95.1, + "real_acc": 98.85, + "fake_acc": 19.46, + "acc": 59.16 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.48, + "fake_acc": 99.7, + "acc": 99.59 + }, + "imle": { + "ap": 99.3, + "auroc": 99.37, + "real_acc": 92.78, + "fake_acc": 98.86, + "acc": 95.82 + }, + "progan": { + "ap": 99.89, + "auroc": 99.87, + "real_acc": 98.93, + "fake_acc": 98.8, + "acc": 98.86 + }, + "san": { + "ap": 98.75, + "auroc": 98.88, + "real_acc": 99.54, + "fake_acc": 65.75, + "acc": 82.65 + }, + "seeingdark": { + "ap": 86.47, + "auroc": 87.81, + "real_acc": 47.22, + "fake_acc": 96.11, + "acc": 71.67 + }, + "stargan": { + "ap": 96.53, + "auroc": 96.59, + "real_acc": 97.1, + "fake_acc": 72.14, + "acc": 84.62 + }, + "stylegan": { + "ap": 94.64, + "auroc": 94.4, + "real_acc": 99.32, + "fake_acc": 51.29, + "acc": 75.3 + }, + "stylegan2": { + "ap": 95.78, + "auroc": 95.66, + "real_acc": 99.16, + "fake_acc": 50.58, + "acc": 74.87 + }, + "whichfaceisreal": { + "ap": 84.4, + "auroc": 85.49, + "real_acc": 100.0, + "fake_acc": 0.0, + "acc": 50.0 + } + }, + "date": "2026-10-27T19:39:25", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step500_ufd8.json b/repro/bench_results/pgc_trained_step500_ufd8.json new file mode 100644 index 0000000000000000000000000000000000000000..8832975b25f13f5c99adf933beda997de0a0a34c --- /dev/null +++ b/repro/bench_results/pgc_trained_step500_ufd8.json @@ -0,0 +1,74 @@ +{ + "tag": "pgc_trained_step500_ufd8", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step500.pth", + "benchmark": "ufd", + "data_root": "/ssdstore/ssalehi/SynthScope/universialfakedetection/test", + "jpeg_fakes": null, + "n_generators": 8, + "means": { + "acc": 91.96, + "map": 98.99, + "mauroc": 98.92 + }, + "per_generator": { + "dalle": { + "ap": 99.0, + "auroc": 98.93, + "real_acc": 98.6, + "fake_acc": 87.2, + "acc": 92.9 + }, + "glide_100_10": { + "ap": 99.12, + "auroc": 99.05, + "real_acc": 98.6, + "fake_acc": 88.9, + "acc": 93.75 + }, + "glide_100_27": { + "ap": 99.08, + "auroc": 99.0, + "real_acc": 98.6, + "fake_acc": 89.2, + "acc": 93.9 + }, + "glide_50_27": { + "ap": 98.99, + "auroc": 98.92, + "real_acc": 98.6, + "fake_acc": 86.7, + "acc": 92.65 + }, + "guided": { + "ap": 98.91, + "auroc": 98.99, + "real_acc": 100.0, + "fake_acc": 59.8, + "acc": 79.9 + }, + "ldm_100": { + "ap": 99.59, + "auroc": 99.52, + "real_acc": 98.6, + "fake_acc": 96.6, + "acc": 97.6 + }, + "ldm_200": { + "ap": 99.7, + "auroc": 99.67, + "real_acc": 98.6, + "fake_acc": 95.9, + "acc": 97.25 + }, + "ldm_200_cfg": { + "ap": 97.49, + "auroc": 97.26, + "real_acc": 98.6, + "fake_acc": 76.9, + "acc": 87.75 + } + }, + "date": "2026-10-28T00:50:55", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step700_forensynths13.json b/repro/bench_results/pgc_trained_step700_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..5c8d4bf707d9ee6379277c9e73551d3123917682 --- /dev/null +++ b/repro/bench_results/pgc_trained_step700_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step700_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step700.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 80.35, + "map": 96.67, + "mauroc": 96.6 + }, + "per_generator": { + "biggan": { + "ap": 99.67, + "auroc": 99.6, + "real_acc": 99.65, + "fake_acc": 94.35, + "acc": 97.0 + }, + "crn": { + "ap": 97.4, + "auroc": 97.69, + "real_acc": 98.46, + "fake_acc": 65.2, + "acc": 81.83 + }, + "cyclegan": { + "ap": 99.86, + "auroc": 99.84, + "real_acc": 99.92, + "fake_acc": 94.1, + "acc": 97.01 + }, + "deepfake": { + "ap": 92.33, + "auroc": 91.79, + "real_acc": 99.96, + "fake_acc": 6.04, + "acc": 53.0 + }, + "gaugan": { + "ap": 99.99, + "auroc": 99.99, + "real_acc": 99.88, + "fake_acc": 99.56, + "acc": 99.72 + }, + "imle": { + "ap": 99.29, + "auroc": 99.32, + "real_acc": 98.46, + "fake_acc": 88.94, + "acc": 93.7 + }, + "progan": { + "ap": 99.98, + "auroc": 99.98, + "real_acc": 99.62, + "fake_acc": 99.52, + "acc": 99.57 + }, + "san": { + "ap": 98.7, + "auroc": 98.78, + "real_acc": 100.0, + "fake_acc": 57.08, + "acc": 78.54 + }, + "seeingdark": { + "ap": 90.62, + "auroc": 91.16, + "real_acc": 97.78, + "fake_acc": 34.44, + "acc": 66.11 + }, + "stargan": { + "ap": 97.11, + "auroc": 97.12, + "real_acc": 99.6, + "fake_acc": 56.73, + "acc": 78.16 + }, + "stylegan": { + "ap": 95.04, + "auroc": 94.65, + "real_acc": 99.68, + "fake_acc": 55.57, + "acc": 77.62 + }, + "stylegan2": { + "ap": 94.65, + "auroc": 94.01, + "real_acc": 99.64, + "fake_acc": 44.94, + "acc": 72.29 + }, + "whichfaceisreal": { + "ap": 92.1, + "auroc": 91.89, + "real_acc": 100.0, + "fake_acc": 0.1, + "acc": 50.05 + } + }, + "date": "2026-10-27T20:11:21", + "synthscope_version": "0.1.0" +} diff --git a/repro/bench_results/pgc_trained_step900_forensynths13.json b/repro/bench_results/pgc_trained_step900_forensynths13.json new file mode 100644 index 0000000000000000000000000000000000000000..935bd180ffaae8489dd263a44811954a866684a7 --- /dev/null +++ b/repro/bench_results/pgc_trained_step900_forensynths13.json @@ -0,0 +1,109 @@ +{ + "tag": "pgc_trained_step900_forensynths13", + "model": "pgc", + "ckpt": "/ssdstore/ssalehi/SynthScope/models/pgc_v1/model_step900.pth", + "benchmark": "forensynths", + "data_root": "/ssdstore/ssalehi/SynthScope/CNNDetection/test", + "jpeg_fakes": null, + "n_generators": 13, + "means": { + "acc": 79.92, + "map": 97.12, + "mauroc": 97.01 + }, + "per_generator": { + "biggan": { + "ap": 99.84, + "auroc": 99.81, + "real_acc": 99.75, + "fake_acc": 96.15, + "acc": 97.95 + }, + "crn": { + "ap": 97.35, + "auroc": 97.64, + "real_acc": 99.2, + "fake_acc": 50.42, + "acc": 74.81 + }, + "cyclegan": { + "ap": 99.92, + "auroc": 99.92, + "real_acc": 99.92, + "fake_acc": 95.84, + "acc": 97.88 + }, + "deepfake": { + "ap": 92.9, + "auroc": 91.94, + "real_acc": 99.96, + "fake_acc": 6.26, + "acc": 53.11 + }, + "gaugan": { + "ap": 100.0, + "auroc": 99.99, + "real_acc": 99.88, + "fake_acc": 99.78, + "acc": 99.83 + }, + "imle": { + "ap": 99.15, + "auroc": 99.17, + "real_acc": 99.2, + "fake_acc": 78.64, + "acc": 88.92 + }, + "progan": { + "ap": 100.0, + "auroc": 100.0, + "real_acc": 99.65, + "fake_acc": 99.83, + "acc": 99.74 + }, + "san": { + "ap": 99.27, + "auroc": 99.34, + "real_acc": 100.0, + "fake_acc": 52.97, + "acc": 76.48 + }, + "seeingdark": { + "ap": 89.92, + "auroc": 90.36, + "real_acc": 97.78, + "fake_acc": 28.33, + "acc": 63.06 + }, + "stargan": { + "ap": 97.88, + "auroc": 97.93, + "real_acc": 99.6, + "fake_acc": 59.58, + "acc": 79.59 + }, + "stylegan": { + "ap": 95.96, + "auroc": 95.57, + "real_acc": 99.77, + "fake_acc": 63.28, + "acc": 81.52 + }, + "stylegan2": { + "ap": 96.07, + "auroc": 95.57, + "real_acc": 99.74, + "fake_acc": 52.32, + "acc": 76.03 + }, + "whichfaceisreal": { + "ap": 94.25, + "auroc": 93.95, + "real_acc": 100.0, + "fake_acc": 0.1, + "acc": 50.05 + } + }, + "date": "2026-10-27T20:43:13", + "synthscope_version": "0.1.0" +} diff --git a/repro/benchmark_specs/omniaid.json b/repro/benchmark_specs/omniaid.json new file mode 100644 index 0000000000000000000000000000000000000000..d78747b43a8ab74e988e4cef52e64c6de795da1b --- /dev/null +++ b/repro/benchmark_specs/omniaid.json @@ -0,0 +1,212 @@ +{ + "model": "omniaid", + "paper": "OmniAID: Decoupling Semantics and Artifacts for Universal AI-Generated Image Detection in the Wild (arXiv 2511.08423)", + "arxiv_id": "2511.08423", + "official_repo": "https://github.com/yunncheng/OmniAID", + "training_set_in_paper": "Two releases matter for the paper: one trained on GenImage SD v1.4 (omniaid_genimage; the academic tables) and one on the authors' Mirage-Train (omniaid_v1 = OmniAID-Mirage). The served key omniaid is the later v2 Mirage-Train release. Only official checkpoints are evaluated; the two-stage training recipe is not reproduced.", + "benchmarks": [ + { + "name": "GenImage (8 subsets) \u2014 GenImage-SD1.4 release", + "datasets": [ + "GenImage" + ], + "metric": "accuracy (mean over subsets)", + "paper_number": { + "acc": 95.94, + "note": "Paper Table 2 (OmniAID): MJ 85.7, SD1.4 98.9, SD1.5 98.8, ADM 91.4, GLIDE 98.7, Wukong 98.1, VQDM 97.3, BigGAN 98.7; mean 95.94 in Tables 6 and 13." + }, + "our_status": "official", + "our_number": { + "note": "omniaid_genimage through SynthScope: 95.82 (mAP 99.02), every subset within 1.5 pt; v1-era 512->336 preprocessing (tag omniaid_genimage_genimage_full, job 265856)." + } + }, + { + "name": "AIGCDetectBenchmark (17) \u2014 GenImage-SD1.4 release", + "datasets": [ + "AIGCDetect" + ], + "metric": "accuracy (mean over subsets)", + "paper_number": { + "acc": 88.87, + "note": "Paper Table 6 (OmniAID, AIGCDetection)." + }, + "our_status": "official", + "our_number": { + "note": "omniaid_genimage through SynthScope: 88.87 (mAP 97.88), exactly the paper's number (tag omniaid_genimage_aigcdetect, job 265856)." + } + }, + { + "name": "Mirage-Test (5 categories) \u2014 GenImage-SD1.4 release", + "datasets": [ + "Mirage-Test" + ], + "metric": "accuracy (mean of Human/Animal/Object/Scene balanced + Anime fake accuracy)", + "paper_number": { + "acc": 51.1, + "note": "Paper Table 3 (OmniAID): Human 59.38, Animal 55.90, Object 49.02, Scene 46.53, Anime 44.67 (fake only)." + }, + "our_status": "official", + "our_number": { + "note": "omniaid_genimage through SynthScope: 51.10, every category equal to the paper (Scene 46.52) (tag omniaid_genimage_mirage_test, job 265935)." + } + }, + { + "name": "GenImage (8 subsets) \u2014 Mirage-Train v1 release (OmniAID-Mirage)", + "datasets": [ + "GenImage" + ], + "metric": "accuracy (mean over subsets)", + "paper_number": { + "acc": 97.24, + "note": "Paper Tables 2 and 6 (OmniAID-Mirage)." + }, + "our_status": "official", + "our_number": { + "note": "omniaid_v1 through SynthScope: 97.16 (mAP 99.20), every subset within 0.6 pt (tag omniaid_v1_genimage_full, job 265857)." + } + }, + { + "name": "AIGCDetectBenchmark (17) \u2014 Mirage-Train v1 release (OmniAID-Mirage)", + "datasets": [ + "AIGCDetect" + ], + "metric": "accuracy (mean over subsets)", + "paper_number": { + "acc": 92.88, + "note": "Paper Tables 6 and 7 (OmniAID-Mirage)." + }, + "our_status": "official", + "our_number": { + "note": "omniaid_v1 through SynthScope: 92.88 (mAP 98.93), exactly the paper's number (tag omniaid_v1_aigcdetect, job 265857)." + } + }, + { + "name": "Mirage-Test (5 categories) \u2014 Mirage-Train v1 release (OmniAID-Mirage)", + "datasets": [ + "Mirage-Test" + ], + "metric": "accuracy (mean of Human/Animal/Object/Scene balanced + Anime fake accuracy)", + "paper_number": { + "acc": 88.39, + "note": "Paper Table 3 (OmniAID-Mirage): Human 93.69, Animal 81.37, Object 84.92, Scene 87.07, Anime 94.92." + }, + "our_status": "official", + "our_number": { + "note": "omniaid_v1 through SynthScope: 88.39, every category equal to the paper (tag omniaid_v1_mirage_test, job 265936)." + } + }, + { + "name": "Chameleon / DRCT-2M", + "datasets": [ + "Chameleon" + ], + "metric": "accuracy", + "paper_number": { + "acc": 77.35, + "note": "Paper Table 6 (GenImage release): Chameleon 77.35, DRCT-2M 88.21." + }, + "our_status": "missing", + "our_number": { + "note": "Chameleon is released by request only; DRCT-2M is not in SynthScope yet." + } + }, + { + "name": "Mirage-Test (v2 release, served)", + "datasets": [ + "Mirage-Test" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 78.46 in the paper's metric (balanced acc 80.37, mAP 96.48) (tag omniaid_mirage_test, job 265937)." + } + }, + { + "name": "SynthWildX (v2 release, served)", + "datasets": [ + "SynthWildX" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 86.02 (mAP 95.52, AUROC 95.31) (tag omniaid_synthwildx, job 265858)." + } + }, + { + "name": "WildRF (v2 release, served)", + "datasets": [ + "WildRF" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 88.36 (mAP 97.28, AUROC 96.91) (tag omniaid_wildrf, job 265858)." + } + }, + { + "name": "BFree-Online (v2 release, served)", + "datasets": [ + "BFree-Online" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 70.26 (mAP 91.70, AUROC 87.18) (tag omniaid_bfree_online, job 265858)." + } + }, + { + "name": "RealChain (v2 release, served)", + "datasets": [ + "RealChain" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 61.71 (mAP 84.02, AUROC 83.63) (tag omniaid_realchain, job 265858)." + } + }, + { + "name": "EvalGEN (v2 release, served)", + "datasets": [ + "EvalGEN" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 74.14 (mAP 96.46, AUROC 93.08) (tag omniaid_evalgen, job 265858)." + } + }, + { + "name": "RRDataset (v2 release, served)", + "datasets": [ + "RRDataset" + ], + "metric": "balanced accuracy", + "paper_number": { + "note": "Not one of the paper's benchmarks." + }, + "our_status": "reference", + "our_number": { + "note": "omniaid (v2) through SynthScope: 71.75 (mAP 84.19, AUROC 82.32) (tag omniaid_rrdataset, job 265858)." + } + } + ] +} diff --git a/repro/benchmark_specs/pgc.json b/repro/benchmark_specs/pgc.json index d9fab74c13e671395ee61680d834732e73fc6fa8..2ee7754cb642956a896ca0a99f4deaee90d8c9c3 100644 --- a/repro/benchmark_specs/pgc.json +++ b/repro/benchmark_specs/pgc.json @@ -1,74 +1,149 @@ { - "model": "pgc", - "paper": "PGC: Peak-Guided Calibration for Generalizable AI-Generated Image Detection (Zhou, Fei, Yu, Xie, Cheng, Xia — ICML 2026 / arXiv 2026)", - "arxiv_id": "2605.21207", - "official_repo": "https://github.com/xiaoyu6868/PGC (ModelScope/HF mirrors). on-disk: /hddstore/ssalehi/SynthScope/PGC_official", - "training_set_in_paper": "Three checkpoints over a frozen DINOv2-Large + LoRA (rank 8). (a) SD-v1.4: GenImage SD-v1.4 split (nature/ai) -> used for GenImage + CommGen15. (b) ProGAN (4-class CNNDet/UniversalFakeDetect train) -> used for UniversalFakeDetect + ForenSynths. (c) joint ProGAN+SD-v1.4 -> used for AIGI(Bench). See README sections 3.1-3.3 and 4.1-4.4 (ckpt->benchmark mapping).", - "benchmarks": [ - { - "name": "GenImage", - "datasets": ["GenImage test"], - "subsets_or_generators": ["8 generators: ADM/adm, BigGAN/biggan, glide, Midjourney, SD-v1.4, SD-v1.5, VQDM/vqdm, wukong"], - "metric": "AP", - "paper_number": {"map": 100.0, "acc": 98.3, "note": "Paper Table 2 (SD-v1.4 checkpoint). +2.1% over prior SOTA. Mean acc ~98.3, mean AP ~100.0."}, - "paper_per_generator": {"note": "Table 2 per-generator acc/AP; SynthScope official-weights run reproduces near-saturated AP=100 on all 8 generators (per_generator all 100.0)."}, - "official_eval_protocol": "PGC_train_sdv1_4_ckpt.pth; DINOv2-Large + LoRA. data/eval_dataset.py reads 0_real/1_fake; transform = PadCenterCrop to 224 (data/transforms.py build_eval_transform, NO global resize -> preserves forensic residual) + DINOv2 normalize (mean .485/.456/.406, std .229/.224/.225). Metric = sklearn average_precision_score / accuracy_score / roc_auc_score (utils/metrics.py). test.py --name test_GenImage.", - "our_status": "reproduced", - "our_number": {"map_official_wts": 100.0, "acc_official_wts": 99.49, "map_synthscope_retrain": 97.62, "acc_synthscope_retrain": 78.60, "ckpt_official": "repro/official_ckpts/pgc/PGC_train_sdv1_4_ckpt.pth; ckpt_synthscope: checkpoints/_train/pgc_genimage/model_0.pth (registry-default)", "tags": "pgc_official_genimage8 / pgc_retrain_genimage8 (pgc_long_g8)"}, - "gap_cause": "Official-weights-in-SynthScope reproduces exactly (mAP 100.0). SynthScope from-scratch retrain reaches 97.62 mAP (within ~2.4 of saturated 100) but acc lags (78.60) because the SynthScope LoRA-DINOv2 retrain is less calibrated than the released checkpoint; longer training gave no gain (recipe-bound, residual cross-generator). Counts as reproduced on the AP headline.", - "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/GenImage_full/test (and test_full). Origin: github.com/GenImage-Dataset/GenImage." - }, - { - "name": "UniversalFakeDetect (UFD)", - "datasets": ["UniversalFakeDetect test"], - "subsets_or_generators": ["~14 test domains (paper Table 5: GANs, deepfakes, low-level-vision, perceptual-loss, guided/LDM/Glide/DALL-E diffusion). SynthScope materializes 8 cells."], - "metric": "AP", - "paper_number": {"map": 98.0, "acc": null, "note": "Paper Table 5 (ProGAN checkpoint). Mean AP ~98.0; +1.7% over prior SOTA."}, - "paper_per_generator": {}, - "official_eval_protocol": "PGC_train_progan_ckpt.pth; PadCenterCrop 224 + DINOv2 norm; AP. test.py --name test_UniversalFakeDetect.", - "our_status": "reproduced", - "our_number": {"map_official_wts": 99.34, "acc_official_wts": 94.16, "map_synthscope_retrain": 99.01, "ckpt_official": "repro/official_ckpts/pgc/PGC_train_progan_ckpt.pth", "tags": "pgc_official_ufd / pgc_retrain_ufd"}, - "gap_cause": "Reproduced (official-wts 99.34, SynthScope retrain 99.01) >= paper 98.0; no gap.", - "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/universialfakedetection/test. Origin: github.com/WisconsinAIVision/UniversalFakeDetect." - }, - { - "name": "ForenSynths (CNNDetection test)", - "datasets": ["CNNDetection / ForenSynths test"], - "subsets_or_generators": ["13-generator CNNDetection test (paper-relevant 8-GAN subset: progan,cyclegan,biggan,gaugan,stargan,stylegan,stylegan2,deepfake)"], - "metric": "AP", - "paper_number": {"map": null, "note": "ForenSynths is reported as a generalization/cross-dataset comparison alongside UFD (often folded into the UniversalFakeDetect table); no separate standalone PGC headline mAP. Listed here as an SynthScope cross-reference."}, - "paper_per_generator": {}, - "official_eval_protocol": "PGC_train_progan_ckpt.pth; PadCenterCrop 224 + DINOv2 norm; AP.", - "our_status": "reproduced", - "our_number": {"map_official_wts": 96.25, "acc_official_wts": 83.73, "map_synthscope_retrain": 89.03, "ckpt_official": "repro/official_ckpts/pgc/PGC_train_progan_ckpt.pth", "tags": "pgc_official_forensynths / pgc_retrain_forensynths"}, - "gap_cause": "Official-weights-in-SynthScope reproduces (96.25 mAP). SynthScope retrain (89.03) below on the GAN generators (ProGAN-trained LoRA-DINOv2 transfers less to cyclegan/gaugan); residual cross-generator gap, not a code defect.", - "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/CNNDetection/test. Origin: github.com/PeterWang512/CNNDetection." - }, - { - "name": "AIGIBench (AIGI)", - "datasets": ["AIGIBench test (HorizonTEL/AIGIBench)"], - "subsets_or_generators": ["25 generators on disk (e.g. DALLE-3, FLUX1-dev, GLIDE, Imagen3, Midjourney, PhotoMaker, InstantID, IP-Adapter, BlendFace, FaceSwap, E4S, InSwap, Infinite-ID, BLIP, CommunityAI ... — spans GAN/diffusion/face-swap/personalization)"], - "metric": "AP", - "paper_number": {"map": 95.0, "acc": 86.6, "note": "Paper Table 4 (joint ProGAN+SD-v1.4 checkpoint). Mean acc ~86.6, mean AP ~95.0; +3.5% over prior SOTA."}, - "paper_per_generator": {}, - "official_eval_protocol": "PGC_train_progan_sdv1_4_ckpt.pth (joint); PadCenterCrop 224 + DINOv2 norm; AP/acc. test.py --name test_AIGI --test_root AIGI/test.", - "our_status": "reproduced", - "our_number": {"map_official_wts": 95.36, "acc_official_wts": 88.33, "map_synthscope_retrain": 86.88, "ckpt_official": "repro/official_ckpts/pgc/PGC_train_progan_sdv1_4_ckpt.pth", "tags": "pgc_official_aigibench25 / pgc_retrain_aigibench"}, - "gap_cause": "Official-weights-in-SynthScope reproduces (95.36 vs paper 95.0). SynthScope retrain (86.88) is the SD-v1.4-only checkpoint (not the joint one), so it trails on the GAN/face-swap subsets; running the joint checkpoint closes it.", - "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/AIGIBench/test (25 generator dirs). Origin: github.com/HorizonTEL/AIGIBench." - }, - { - "name": "CommGen15 (commercial in-the-wild)", - "datasets": ["CommGen15 (this paper's released benchmark)"], - "subsets_or_generators": ["15 commercial generative models (e.g. Sora, Kling, Google Imagen, Flux, and other commercial T2I/T2V services)"], - "metric": "AP", - "paper_number": {"map": null, "acc": 97.9, "note": "Paper Table 3 (SD-v1.4 checkpoint). Mean acc ~97.9; +12.3% over best baseline — the headline gain. CommGen15 = the paper's own challenging commercial benchmark."}, - "paper_per_generator": {}, - "official_eval_protocol": "PGC_train_sdv1_4_ckpt.pth; PadCenterCrop 224 + DINOv2 norm; AP/acc. test.py --name test_CommGen15 --test_root CommGen15.", - "our_status": "reproduced", - "our_number": {"map_official_wts": 99.26, "acc_official_wts": 97.39, "map_synthscope_retrain": 98.61, "acc_synthscope_retrain": 66.66, "ckpt_official": "repro/official_ckpts/pgc/PGC_train_sdv1_4_ckpt.pth", "tags": "pgc_official_commgen15 / pgc_retrain_commgen15"}, - "gap_cause": "Official-weights-in-SynthScope reproduces the AP (99.26) and acc 97.39 ~= paper 97.9. SynthScope retrain matches on AP (98.61) but acc collapses (66.66) due to threshold/calibration drift in the from-scratch LoRA head; the released checkpoint's peak-guided calibration is what the paper headline relies on.", - "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/CommGen15/test. Origin: ModelScope datasets/xiaoyuzhou68/CommGen15 and HF datasets/xiaoyuzhou68/CommGen15 (officially released by the paper)." - } - ] + "model": "pgc", + "paper": "PGC: Peak-Guided Calibration for Generalizable AI-Generated Image Detection (Zhou, Fei, Yu, Xie, Cheng, Xia \u2014 ICML 2026 / arXiv 2026)", + "arxiv_id": "2605.21207", + "official_repo": "https://github.com/xiaoyu6868/PGC (ModelScope/HF mirrors). on-disk: /hddstore/ssalehi/SynthScope/PGC_official", + "training_set_in_paper": "Three checkpoints over a frozen DINOv2-Large + LoRA (rank 8). (a) SD-v1.4: GenImage SD-v1.4 split (nature/ai) -> used for GenImage + CommGen15. (b) ProGAN (4-class CNNDet/UniversalFakeDetect train) -> used for UniversalFakeDetect + ForenSynths. (c) joint ProGAN+SD-v1.4 -> used for AIGI(Bench). See README sections 3.1-3.3 and 4.1-4.4 (ckpt->benchmark mapping).", + "benchmarks": [ + { + "name": "GenImage", + "datasets": [ + "GenImage test" + ], + "subsets_or_generators": [ + "8 generators: ADM/adm, BigGAN/biggan, glide, Midjourney, SD-v1.4, SD-v1.5, VQDM/vqdm, wukong" + ], + "metric": "AP", + "paper_number": { + "map": 100.0, + "acc": 98.3, + "note": "Paper Table 2 (SD-v1.4 checkpoint). +2.1% over prior SOTA. Mean acc ~98.3, mean AP ~100.0." + }, + "paper_per_generator": { + "note": "Table 2 per-generator acc/AP; SynthScope official-weights run reproduces near-saturated AP=100 on all 8 generators (per_generator all 100.0)." + }, + "official_eval_protocol": "PGC_train_sdv1_4_ckpt.pth; DINOv2-Large + LoRA. data/eval_dataset.py reads 0_real/1_fake; transform = PadCenterCrop to 224 (data/transforms.py build_eval_transform, NO global resize -> preserves forensic residual) + DINOv2 normalize (mean .485/.456/.406, std .229/.224/.225). Metric = sklearn average_precision_score / accuracy_score / roc_auc_score (utils/metrics.py). test.py --name test_GenImage.", + "our_status": "reproduced", + "our_number": { + "map_official_wts": 100.0, + "acc_official_wts": 99.49, + "map_synthscope_retrain": 97.62, + "acc_synthscope_retrain": 78.6, + "ckpt_official": "repro/official_ckpts/pgc/PGC_train_sdv1_4_ckpt.pth; ckpt_synthscope: checkpoints/_train/pgc_genimage/model_0.pth (registry-default)", + "tags": "pgc_official_genimage8 / pgc_retrain_genimage8 (pgc_long_g8)" + }, + "gap_cause": "Official-weights-in-SynthScope reproduces exactly (mAP 100.0). SynthScope from-scratch retrain reaches 97.62 mAP (within ~2.4 of saturated 100) but acc lags (78.60) because the SynthScope LoRA-DINOv2 retrain is less calibrated than the released checkpoint; longer training gave no gain (recipe-bound, residual cross-generator). Counts as reproduced on the AP headline.", + "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/GenImage_full/test (and test_full). Origin: github.com/GenImage-Dataset/GenImage." + }, + { + "name": "UniversalFakeDetect (UFD)", + "datasets": [ + "UniversalFakeDetect test" + ], + "subsets_or_generators": [ + "~14 test domains (paper Table 5: GANs, deepfakes, low-level-vision, perceptual-loss, guided/LDM/Glide/DALL-E diffusion). SynthScope materializes 8 cells." + ], + "metric": "AP", + "paper_number": { + "map": 98.0, + "acc": null, + "note": "Paper Table 5 (ProGAN checkpoint). Mean AP ~98.0; +1.7% over prior SOTA." + }, + "paper_per_generator": {}, + "official_eval_protocol": "PGC_train_progan_ckpt.pth; PadCenterCrop 224 + DINOv2 norm; AP. test.py --name test_UniversalFakeDetect.", + "our_status": "reproduced", + "our_number": { + "map_official_wts": 99.34, + "acc_official_wts": 94.16, + "map_synthscope_retrain": 98.99, + "ckpt_official": "repro/official_ckpts/pgc/PGC_train_progan_ckpt.pth", + "tags": "pgc_official_ufd / pgc_trained_step500_ufd8 (official recipe in SynthScope, experiments/image/pgc.sh)", + "acc_synthscope_retrain": 91.96 + }, + "gap_cause": "Reproduced (official weights 99.34; the official recipe trained in SynthScope, step 500: 98.99) >= paper 98.0.", + "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/universialfakedetection/test. Origin: github.com/WisconsinAIVision/UniversalFakeDetect." + }, + { + "name": "ForenSynths (CNNDetection test)", + "datasets": [ + "CNNDetection / ForenSynths test" + ], + "subsets_or_generators": [ + "13-generator CNNDetection test (paper-relevant 8-GAN subset: progan,cyclegan,biggan,gaugan,stargan,stylegan,stylegan2,deepfake)" + ], + "metric": "AP", + "paper_number": { + "map": null, + "note": "ForenSynths is reported as a generalization/cross-dataset comparison alongside UFD (often folded into the UniversalFakeDetect table); no separate standalone PGC headline mAP. Listed here as an SynthScope cross-reference." + }, + "paper_per_generator": {}, + "official_eval_protocol": "PGC_train_progan_ckpt.pth; PadCenterCrop 224 + DINOv2 norm; AP.", + "our_status": "reproduced", + "our_number": { + "map_official_wts": 96.25, + "acc_official_wts": 83.73, + "map_synthscope_retrain": 97.41, + "ckpt_official": "repro/official_ckpts/pgc/PGC_train_progan_ckpt.pth", + "tags": "pgc_official_forensynths / pgc_trained_step500_forensynths13 (FS-8 subset of the 13)", + "acc_synthscope_retrain": 85.89 + }, + "gap_cause": "Not a paper number. The official recipe trained in SynthScope (step 500) gives FS-8 97.41 mAP, above the official checkpoint (96.41 on the same evaluation); the earlier 89.03 retrain overshot the early transfer peak.", + "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/CNNDetection/test. Origin: github.com/PeterWang512/CNNDetection." + }, + { + "name": "AIGIBench (AIGI)", + "datasets": [ + "AIGIBench test (HorizonTEL/AIGIBench)" + ], + "subsets_or_generators": [ + "25 generators on disk (e.g. DALLE-3, FLUX1-dev, GLIDE, Imagen3, Midjourney, PhotoMaker, InstantID, IP-Adapter, BlendFace, FaceSwap, E4S, InSwap, Infinite-ID, BLIP, CommunityAI ... \u2014 spans GAN/diffusion/face-swap/personalization)" + ], + "metric": "AP", + "paper_number": { + "map": 95.0, + "acc": 86.6, + "note": "Paper Table 4 (joint ProGAN+SD-v1.4 checkpoint). Mean acc ~86.6, mean AP ~95.0; +3.5% over prior SOTA." + }, + "paper_per_generator": {}, + "official_eval_protocol": "PGC_train_progan_sdv1_4_ckpt.pth (joint); PadCenterCrop 224 + DINOv2 norm; AP/acc. test.py --name test_AIGI --test_root AIGI/test.", + "our_status": "reproduced", + "our_number": { + "map_official_wts": 95.36, + "acc_official_wts": 88.33, + "map_synthscope_retrain": 86.88, + "ckpt_official": "repro/official_ckpts/pgc/PGC_train_progan_sdv1_4_ckpt.pth", + "tags": "pgc_official_aigibench25 / pgc_retrain_aigibench" + }, + "gap_cause": "Official-weights-in-SynthScope reproduces (95.36 vs paper 95.0). SynthScope retrain (86.88) is the SD-v1.4-only checkpoint (not the joint one), so it trails on the GAN/face-swap subsets; running the joint checkpoint closes it.", + "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/AIGIBench/test (25 generator dirs). Origin: github.com/HorizonTEL/AIGIBench." + }, + { + "name": "CommGen15 (commercial in-the-wild)", + "datasets": [ + "CommGen15 (this paper's released benchmark)" + ], + "subsets_or_generators": [ + "15 commercial generative models (e.g. Sora, Kling, Google Imagen, Flux, and other commercial T2I/T2V services)" + ], + "metric": "AP", + "paper_number": { + "map": null, + "acc": 97.9, + "note": "Paper Table 3 (SD-v1.4 checkpoint). Mean acc ~97.9; +12.3% over best baseline \u2014 the headline gain. CommGen15 = the paper's own challenging commercial benchmark." + }, + "paper_per_generator": {}, + "official_eval_protocol": "PGC_train_sdv1_4_ckpt.pth; PadCenterCrop 224 + DINOv2 norm; AP/acc. test.py --name test_CommGen15 --test_root CommGen15.", + "our_status": "reproduced", + "our_number": { + "map_official_wts": 99.26, + "acc_official_wts": 97.39, + "map_synthscope_retrain": 98.61, + "acc_synthscope_retrain": 66.66, + "ckpt_official": "repro/official_ckpts/pgc/PGC_train_sdv1_4_ckpt.pth", + "tags": "pgc_official_commgen15 / pgc_retrain_commgen15" + }, + "gap_cause": "Official-weights-in-SynthScope reproduces the AP (99.26) and acc 97.39 ~= paper 97.9. SynthScope retrain matches on AP (98.61) but acc collapses (66.66) due to threshold/calibration drift in the from-scratch LoRA head; the released checkpoint's peak-guided calibration is what the paper headline relies on.", + "dataset_source": "on-disk /hddstore/ssalehi/SynthScope/CommGen15/test. Origin: ModelScope datasets/xiaoyuzhou68/CommGen15 and HF datasets/xiaoyuzhou68/CommGen15 (officially released by the paper)." + } + ] } diff --git a/repro/build_wild_table.py b/repro/build_wild_table.py index aa87f7d134371fea8c5f5857bb6585bfb40147e5..34ac404dcb5815c18d69f00f6451a80e313f3f3d 100644 --- a/repro/build_wild_table.py +++ b/repro/build_wild_table.py @@ -28,7 +28,7 @@ DATASETS = [ # (config name, column title, what the set is, caveat) ] # Display names, in the order of the README's detector table. -DETECTORS = [("cure", "CuRe"), ("dda", "DDA"), ("d3qe", "D3QE"), ("spai", "SPAI"), ("dinov3", "DINOv3 probe"), ("co_spy", "Co-Spy"), ("aide", "AIDE"), +DETECTORS = [("cure", "CuRe"), ("dda", "DDA"), ("omniaid", "OmniAID (v2)"), ("d3qe", "D3QE"), ("spai", "SPAI"), ("dinov3", "DINOv3 probe"), ("co_spy", "Co-Spy"), ("aide", "AIDE"), ("vib_net", "VIB-Net"), ("pgc", "PGC"), ("npr", "NPR"), ("safe", "SAFE"), ("fat_former", "FatFormer"), ("clip", "UnivFD / CLIP"), ("cnn", "CNNDetection"), ("dinov2", "DINOv2 probe"), ("adapted_dino", "Adapted DINOv2")] diff --git a/tests/test_omniaid_smoke.py b/tests/test_omniaid_smoke.py new file mode 100644 index 0000000000000000000000000000000000000000..d7eb86923ce94f7d1d227310ac85fac3584222db --- /dev/null +++ b/tests/test_omniaid_smoke.py @@ -0,0 +1,80 @@ +"""Smoke test for the OmniAID integration. + +Builds the real model for the three released configurations, so it needs the CLIP ViT-L/14@336 weights +(HF ``openai/clip-vit-large-patch14-336``, downloaded on first use). Checks: +* the state-dict layout matches each release's expert configuration (and the released checkpoint, if + ``$SYNTHSCOPE_CKPT_DIR/.pth`` exists: same keys and shapes), +* the registry transforms give [3, 336, 336] tensors (v2: one resize; v1-era: 512 then 336), +* ``forward`` -> [B, 2] finite logits; ``compute_loss`` + backward reach the experts, router and head, while the + frozen CLIP feature extractor and the shared main weights get no gradient. +""" + +from __future__ import annotations + +import os +import sys + +import torch +from hydra import compose, initialize_config_dir +from PIL import Image + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path[:0] = [ROOT, os.path.join(ROOT, "interface")] + +from models.image.omniaid.omniaid import SynthScopeOmniAIDModel # noqa: E402 +from backend.registry import get_spec # noqa: E402 + +EXPECTED = {"omniaid": (6, 1), "omniaid_v1": (6, 8), "omniaid_genimage": (3, 4)} # (experts, rank per expert) + + +def build(key: str) -> SynthScopeOmniAIDModel: + with initialize_config_dir(config_dir=os.path.join(ROOT, "conf"), version_base=None): + cfg = compose(config_name="config", overrides=[f"model@ModelConfig=image/{key}"]) + return SynthScopeOmniAIDModel(cfg) + + +def test_layout(key: str, model) -> None: + experts, rank = EXPECTED[key] + sd = {k: v for k, v in model.state_dict().items() if not k.startswith("criterion")} + u = sd["encoder_layers.0.self_attn.q_proj.U_experts.0"] + assert tuple(u.shape) == (1024, rank), u.shape + assert f"encoder_layers.0.self_attn.q_proj.U_experts.{experts - 1}" in sd + ckpt = os.path.join(os.environ.get("SYNTHSCOPE_CKPT_DIR", ""), f"{key}.pth") + if os.path.isfile(ckpt): + released = torch.load(ckpt, map_location="cpu", weights_only=False)["model"] + assert set(released) == set(sd) and all(released[k].shape == sd[k].shape for k in sd) + print(f" {key}: released checkpoint matches the layout ({len(sd)} tensors)") + + +def test_transforms() -> None: + for key in EXPECTED: + tf = get_spec(key).transform + for size in [(200, 250), (1024, 768)]: + x = tf(Image.new("RGB", size, (128, 64, 32))) + assert tuple(x.shape) == (3, 336, 336) and torch.isfinite(x).all() + + +def test_forward_and_loss(model) -> None: + images = torch.randn(2, 3, 336, 336) + model.eval() + with torch.no_grad(): + logits = model(images) + assert tuple(logits.shape) == (2, 2) and torch.isfinite(logits).all() + model.train() + model.zero_grad() + model.compute_loss((images, torch.tensor([0, 1]), torch.tensor([0, 0])))["loss"].backward() + grads = {n for n, p in model.named_parameters() if p.grad is not None} + assert "head.weight" in grads and any("experts" in n for n in grads) + assert not any(n.startswith("feature_extractor.") or n.endswith("weight_main") for n in grads) + + +if __name__ == "__main__": + print("[B] registry transforms ...") + test_transforms() + for key in EXPECTED: + print(f"[A/C] {key}: layout, forward, loss ...") + m = build(key) + test_layout(key, m) + test_forward_and_loss(m) + del m + print("OmniAID smoke test passed") diff --git a/trainer.py b/trainer.py index ebdcab09ee27aa8eb0dd98c22321f17b47bfa8cd..bbdde6c0ed7b8d8a63b853a99b1423dc9f441d8c 100644 --- a/trainer.py +++ b/trainer.py @@ -13,6 +13,7 @@ from models.image.npr.npr import NPR from models.image.pgc.pgc import SynthScopePGCModel from models.image.cure.cure import SynthScopeCuReModel from models.image.dda.dda import SynthScopeDDAModel +from models.image.omniaid.omniaid import SynthScopeOmniAIDModel from models.image.d3qe.d3qe import SynthScopeD3QEModel from models.video.d3.d3 import SynthScopeD3Model from models.video.restrav.restrav import SynthScopeReStraVModel @@ -49,7 +50,9 @@ class SynthScopeModelTrainer: self.use_metrics = len(config.metrics) > 0 self.scheduler = None warmup = config.TrainerConfig.warmup - if warmup.mode != "off": + # A bare `mode: off` in YAML is the boolean False (and `on` is True), so compare the string form: before this, an + # unquoted `off` (conf/trainer/default.yaml, aide, restrav, ...) silently enabled a schedule annealing the LR to 0. + if str(warmup.mode).lower() not in ("off", "false", "none"): num_training_steps = config.TrainerConfig.epochs * (len(self.train_loader) // self.accelerator.gradient_accumulation_steps) self.scheduler = get_scheduler(warmup.lr_scheduler, optimizer=self.optimizer, num_warmup_steps=int(warmup.warmup_ratio * num_training_steps), num_training_steps=num_training_steps) self.model, self.optimizer, self.scheduler, self.train_loader, self.val_loader, self.test_loader = self.accelerator.prepare( @@ -90,6 +93,8 @@ class SynthScopeModelTrainer: return SynthScopeCuReModel(config) if model_type == "dda": return SynthScopeDDAModel(config) + if model_type == "omniaid": + return SynthScopeOmniAIDModel(config) if model_type == "d3qe": return SynthScopeD3QEModel(config) if model_type == "d3": @@ -228,7 +233,127 @@ class SynthScopeModelTrainer: self._save_model(epoch, self.config.TrainerConfig.save_path) self.accelerator.print(f"Epoch {epoch}, Model Saved") + @staticmethod + def _val_accuracy(metrics_dict): + """Mean of the real and fake accuracies over the validation folders (balanced accuracy; on a balanced + validation set it equals the plain accuracy the CNNDetection-family training scripts monitor).""" + real = [v for k, v in metrics_dict.items() if k.startswith("real_accuracy")] + fake = [v for k, v in metrics_dict.items() if k.startswith("fake_accuracy")] + if not real or not fake: + raise ValueError("the plateau schedule needs metrics=[real_accuracy,fake_accuracy]") + return (sum(real) / len(real) + sum(fake) / len(fake)) / 2 + + def _train_with_plateau(self, plateau): + """The CNNDetection-family schedule (CNNDetection, D3QE, ...; their EarlyStopping + adjust_learning_rate): + validate after every epoch and keep the checkpoint with the best validation accuracy (``model_best.pth``); + when it has not improved by at least ``delta`` for ``patience`` epochs, divide the learning rate by + ``factor`` and restart the tracker with ``delta_after_drop`` (so, as in the official code, the next epoch is + kept); stop once the learning rate falls below ``min_lr`` or after ``epochs`` epochs.""" + cfg = self.config.TrainerConfig + best, bad, delta = None, 0, float(plateau.delta) + for epoch in range(cfg.epochs): + train_loss, train_metrics_dict = self.train_epoch() + self.accelerator.print(f"Epoch {epoch}, Train Loss: {train_loss}, Train Metrics: {train_metrics_dict}") + self.logger.log_metrics({"train/loss": train_loss, **{f"train/{m}": v for m, v in train_metrics_dict.items()}}, self.global_step) + val_loss, val_metrics_dict = self.val() + acc = self._val_accuracy(val_metrics_dict) + lr = self.optimizer.param_groups[0]["lr"] + self.accelerator.print(f"Epoch {epoch}, Val accuracy {acc:.4f} (lr {lr:.2e}, step {self.global_step})") + self.logger.log_metrics({"val/accuracy": acc, "lr": lr}, self.global_step) + if cfg.save: + self._save_model("last", cfg.save_path) + if best is None or acc >= best + delta: + best, bad = acc, 0 + if cfg.save: + self._save_model("best", cfg.save_path) + self.accelerator.print(f"Epoch {epoch}: best validation accuracy {acc:.4f}, saved model_best.pth") + continue + bad += 1 + if bad < int(plateau.patience): + continue + for group in self.optimizer.param_groups: + group["lr"] /= float(plateau.factor) + new_lr = self.optimizer.param_groups[0]["lr"] + if new_lr < float(plateau.min_lr): + self.accelerator.print(f"Epoch {epoch}: learning rate {new_lr:.2e} below min_lr, early stopping") + break + self.accelerator.print(f"Epoch {epoch}: no improvement for {bad} epochs, learning rate -> {new_lr:.2e}") + best, bad, delta = None, 0, float(plateau.get("delta_after_drop", plateau.delta)) + self.logger.finish() + + def _train_with_lr_step(self, step): + """A fixed step schedule as in NPR's train.py (``delr_freq``): after epoch ``e`` (0-based) with + ``e % every == 0 and e != 0`` the learning rate is multiplied by ``factor``; validation after every epoch is + logged (plus ``model_best.pth`` by validation accuracy), and the final model is ``model_last.pth``, which is + the checkpoint the official code releases.""" + cfg = self.config.TrainerConfig + best = None + for epoch in range(cfg.epochs): + train_loss, train_metrics_dict = self.train_epoch() + self.accelerator.print(f"Epoch {epoch}, Train Loss: {train_loss}, Train Metrics: {train_metrics_dict}") + self.logger.log_metrics({"train/loss": train_loss, **{f"train/{m}": v for m, v in train_metrics_dict.items()}}, self.global_step) + if epoch % int(step.every) == 0 and epoch != 0: + for group in self.optimizer.param_groups: + group["lr"] *= float(step.factor) + self.accelerator.print(f"Epoch {epoch}: learning rate -> {self.optimizer.param_groups[0]['lr']:.3e}") + val_loss, val_metrics_dict = self.val() + acc = self._val_accuracy(val_metrics_dict) + self.accelerator.print(f"Epoch {epoch}, Val accuracy {acc:.4f} (lr {self.optimizer.param_groups[0]['lr']:.3e}, step {self.global_step})") + self.logger.log_metrics({"val/accuracy": acc, "lr": self.optimizer.param_groups[0]["lr"]}, self.global_step) + if cfg.save: + self._save_model("last", cfg.save_path) + if best is None or acc > best: + best = acc + self._save_model("best", cfg.save_path) + self.logger.finish() + + def _train_with_step_budget(self, budget): + """A fixed budget of micro-batches, as in PGC's train.py: the loss of each micro-batch is divided by + ``grad_accum_steps`` and the optimizer steps every ``grad_accum_steps`` micro-batches, after clipping the gradient + norm to ``max_grad_norm`` (0: no clipping); ``model_step.pth`` is saved every ``save_every`` micro-batches, up + to ``max_steps``. For detectors whose transfer peaks early (PGC's released checkpoint is ~400 steps in), the + saved curve is evaluated rather than only the last model.""" + accum = int(budget.get("grad_accum_steps", 1)) + clip = float(budget.get("max_grad_norm", 0.0)) + max_steps, save_every = int(budget.max_steps), int(budget.save_every) + save_path = self.config.TrainerConfig.save_path + step, running = 0, 0.0 + self.model.train() + self.optimizer.zero_grad() + while step < max_steps: + for batch in tqdm(self.train_loader, desc="Training"): + loss = self.train_step(batch) + self.accelerator.backward(loss / accum) + running += loss.detach().item() + step += 1 + if step % accum == 0: + if clip > 0: + self.accelerator.clip_grad_norm_(self.model.parameters(), clip) + self.optimizer.step() + if self.scheduler is not None: + self.scheduler.step() + self.optimizer.zero_grad() + self.global_step += 1 + if step % save_every == 0: + self.accelerator.print(f"step {step} (optimizer step {self.global_step}): mean loss {running / save_every:.4f}, " + f"lr {self.optimizer.param_groups[0]['lr']:.3e}") + self.logger.log_metrics({"train/loss": running / save_every}, step) + running = 0.0 + self._save_model(f"step{step}", save_path) + if step >= max_steps: + break + self.logger.finish() + def train(self): + plateau = self.config.TrainerConfig.get("plateau", None) + if plateau is not None: + return self._train_with_plateau(plateau) + lr_step = self.config.TrainerConfig.get("lr_step", None) + if lr_step is not None: + return self._train_with_lr_step(lr_step) + budget = self.config.TrainerConfig.get("step_budget", None) + if budget is not None: + return self._train_with_step_budget(budget) for epoch in range(self.config.TrainerConfig.epochs): if epoch % self.config.TrainerConfig.val_interval == 0: val_loss, val_metrics_dict = self.val() diff --git a/transformations.py b/transformations.py index 7e11ce57380d1f9e1eda66a64a95c1feb4584088..8748e3ebc783d49aebab434a9b0d08881ac45172 100644 --- a/transformations.py +++ b/transformations.py @@ -1,10 +1,55 @@ +from io import BytesIO from typing import Callable +import cv2 +import numpy as np import torchvision.transforms.v2 as v2 +from PIL import Image +from scipy.ndimage import gaussian_filter from torchvision import tv_tensors import torch import random +class CNNDetAugment: + """The blur + JPEG augmentation of the CNNDetection code base (``data_augment``), which CNNDetection, D3QE and other + detectors built on it train with: with probability ``blur_prob`` a Gaussian blur (scipy, sigma drawn from + ``blur_sig``, in place per channel), then with probability ``jpg_prob`` a JPEG round trip (cv2 or PIL, quality drawn + from ``jpg_qual``). Ops copied from D3QE's data/process.py; takes and returns a uint8 [3, H, W] tensor.""" + + def __init__(self, blur_prob, blur_sig, jpg_prob, jpg_method, jpg_qual): + self.blur_prob, self.blur_sig = blur_prob, list(blur_sig) + self.jpg_prob, self.jpg_method, self.jpg_qual = jpg_prob, list(jpg_method), list(jpg_qual) + + @staticmethod + def _sample_continuous(s): + return s[0] if len(s) == 1 else random.random() * (s[1] - s[0]) + s[0] + + @staticmethod + def _cv2_jpg(img, quality): + _, encimg = cv2.imencode(".jpg", img[:, :, ::-1], [int(cv2.IMWRITE_JPEG_QUALITY), quality]) + return cv2.imdecode(encimg, 1)[:, :, ::-1] + + @staticmethod + def _pil_jpg(img, quality): + out = BytesIO() + Image.fromarray(img).save(out, format="jpeg", quality=quality) + img = np.array(Image.open(out)) + out.close() + return img + + def __call__(self, image): + img = np.ascontiguousarray(image.permute(1, 2, 0).numpy() if isinstance(image, torch.Tensor) else np.array(image)) + if random.random() < self.blur_prob: + sig = self._sample_continuous(self.blur_sig) + for c in range(3): + gaussian_filter(img[:, :, c], output=img[:, :, c], sigma=sig) + if random.random() < self.jpg_prob: + method = random.choice(self.jpg_method) + quality = random.choice(self.jpg_qual) + img = self._cv2_jpg(img, quality) if method == "cv2" else self._pil_jpg(img, quality) + return torch.from_numpy(np.ascontiguousarray(img)).permute(2, 0, 1).contiguous() + + class RandomMask: def __init__(self, ratio=(0.0, 0.75), patch_size=16, p=0.5): self.ratio = ratio @@ -40,6 +85,8 @@ class AIGITransformations: def _create_transform(self, transform_name: str): if transform_name == "resize_224": return v2.Resize((224, 224)) + elif transform_name == "resize_256": # square, as NPR/FatFormer/CNNDetection-style training (Resize((256, 256))) + return v2.Resize((256, 256)) elif transform_name == "resize_short_256": return v2.Resize(256) elif transform_name == "resize_short_384": @@ -66,6 +113,10 @@ class AIGITransformations: return v2.RandomHorizontalFlip(p=0.5) elif transform_name == "random_vertical_flip": return v2.RandomVerticalFlip(p=0.5) + elif transform_name == "blur_jpeg_cnndet": # CNNDetection/D3QE training: blur 0.1 (sigma 0-3), JPEG 0.1 (cv2/pil, q30-100) + return CNNDetAugment(0.1, (0.0, 3.0), 0.1, ("cv2", "pil"), range(30, 101)) + elif transform_name == "blur_jpeg_cnndet_val": # the same on the validation split, fixed: sigma 1.5, PIL q65 + return CNNDetAugment(0.1, (1.5,), 0.1, ("pil",), (65,)) elif transform_name == "jpeg_compression": return v2.RandomApply([v2.JPEG(quality=(30, 100))], p=0.5) elif transform_name == "random_crop_224":