diff --git a/LEADERBOARD.md b/LEADERBOARD.md index 4217b88b31cf03cfb837fc657e2a2762a5596fc9..1b72d0bc12d56e86e94d48010c61d2b080537183 100644 --- a/LEADERBOARD.md +++ b/LEADERBOARD.md @@ -34,3 +34,5 @@ Metrics are percentages. **mAP** = mean AP over generators. Each track is ranked | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1 | SPLIT | training-free | 99.41 | 99.79 | — | 99.97 | ✓ | Training-free X-CLIP patch roughness x motion incoherence; Acc at the real-only FPR-1% calibration | | 2 | D3 | training-free | 95.53 | 68.84 | — | 98.45 | ✓ | Training-free X-CLIP 2nd-order feature volatility (official preprocessing) | +| 3 | STALL | training-free | 56.89 | 32.05 | — | 82.59 | ✓ | Training-free DINOv3 spatio-temporal likelihood under VATEX real-video statistics | +| 4 | ReStraV | frozen-probe | 30.01 | 33.60 | — | 43.06 | ✓ | Frozen DINOv2 trajectory geometry + MLP trained on GenVidBench; does not transfer to GenVideo-Val (AUROC below 50) | diff --git a/benchmark/README.md b/benchmark/README.md index f6108538f51c7a3b2983e13efe05f60c7ca85cb6..ae7749d86dcc3cb232920068d6e64222f72ff2f5 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -19,7 +19,7 @@ from the submission files in [`submissions/`](submissions/) — see [LEADERBOARD | `genimage-sdv1.4` | image | GenImage SD-v1.4 real/fake | in-distribution diffusion | **AUROC** | | `genvideo-val` | video | GenVideo-Val: all 10,000 MSR-VTT reals against every fake, per generator | mixed | **AUROC** (AP with fake as positive is unstable here: as few as 56 fakes against 10,000 reals) | -**These three tracks are only the start.** The repo supports 13 image and 8 video benchmark datasets (see the table in the +**These three tracks are only the start.** The repo supports 20 image and 8 video benchmark datasets (see the table in the [README](../README.md#-benchmarks)), and the web page's Leaderboard tab already compares detectors on 30+ benchmark groups. A dataset becomes a standardized track once its protocol (test set, grouping, metric) is fixed, so results stay comparable; adding one means a schema entry, an `evaluate.py` track and verified submissions, described in diff --git a/benchmark/bench_eval.py b/benchmark/bench_eval.py index d28120f0360eba338e03d730e2b47d7322d1688b..f63066d7229e8552f770195db711c469314e1cf0 100644 --- a/benchmark/bench_eval.py +++ b/benchmark/bench_eval.py @@ -144,8 +144,9 @@ def discover(benchmark: str, root: Path, shared_reals: Path | None, select: set[ def native_res_transform(tf): - """Drop the up-scaling Resize/CenterCrop from a model's eval transform, keeping - ToTensor + Normalize, so images are scored at NATIVE resolution. + """Drop the up-scaling resize/center-crop steps from a model's eval transform, keeping + to_dtype + normalize, so images are scored at NATIVE resolution (ValueError for eval lists + whose own steps resize, see ``AIGITransformations.without_resize``). Some methods (e.g. NPR on GANGen-Detection/freqnet) detect up-sampling artifacts and MUST NOT be fed a bilinearly up-scaled image — resizing a 128x128 GAN crop to 256x256 @@ -153,11 +154,7 @@ def native_res_transform(tf): GANGen recipe runs with ``no_resize=True``; this reproduces it. Safe only when every image in a per-generator batch shares one resolution (true for freqnet's per-gen sets), since bench_eval stacks a batch into a single tensor.""" - import torchvision.transforms as T - if not isinstance(tf, T.Compose): - return tf - kept = [t for t in tf.transforms if not isinstance(t, (T.Resize, T.CenterCrop))] - return T.Compose(kept) + return tf.without_resize() @torch.no_grad() diff --git a/benchmark/submissions/restrav.json b/benchmark/submissions/restrav.json index 8366417b79eb03c5e7205f06920886eaa35e3367..f3b356c29a903d5256e2333cf8eaa4009c2a51b9 100644 --- a/benchmark/submissions/restrav.json +++ b/benchmark/submissions/restrav.json @@ -3,20 +3,61 @@ "track": "genvideo-val", "modality": "video", "model_type": "restrav", - "paper": "2025", + "paper": "Interno et al., 2025", "url": "https://github.com/SMSD75/SynthScope", "evaluation": "frozen-probe", "metrics": { - "acc": null, - "map": null, - "ap": 96.67, - "auroc": 97.06 + "acc": 30.01, + "map": 33.6, + "ap": null, + "auroc": 43.06 + }, + "per_generator": { + "Crafter": { + "auroc": 89.15, + "ap": 68.75 + }, + "Gen2": { + "auroc": 2.76, + "ap": 6.37 + }, + "HotShot": { + "auroc": 0.1, + "ap": 3.35 + }, + "Lavie": { + "auroc": 34.51, + "ap": 25.86 + }, + "ModelScope": { + "auroc": 78.96, + "ap": 59.16 + }, + "MoonValley": { + "auroc": 2.07, + "ap": 3.02 + }, + "MorphStudio": { + "auroc": 92.58, + "ap": 74.66 + }, + "Show_1": { + "auroc": 91.56, + "ap": 66.49 + }, + "Sora": { + "auroc": 0.26, + "ap": 0.29 + }, + "WildScrape": { + "auroc": 38.7, + "ap": 28.08 + } }, - "per_generator": null, "submitted_by": "SynthScope", - "date": "2026-06-02", + "date": "2026-11-02", "verified": true, - "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md", - "protocol_notes": "Frozen DINOv2 trajectory geometry + MLP", + "source": "repro/bench_results/restrav_genvideo_full.json (benchmark/bench_eval_video.py --dataset genvideo, no --limit)", + "protocol_notes": "Frozen DINOv2 trajectory geometry + MLP trained on GenVidBench; does not transfer to GenVideo-Val (AUROC below 50)", "synthscope_version": "0.1.0" } diff --git a/benchmark/submissions/stall.json b/benchmark/submissions/stall.json new file mode 100644 index 0000000000000000000000000000000000000000..b66fd06b6df93a828a7faba76311cf05572438d8 --- /dev/null +++ b/benchmark/submissions/stall.json @@ -0,0 +1,63 @@ +{ + "method": "STALL", + "track": "genvideo-val", + "modality": "video", + "model_type": "stall", + "paper": "Ben Hayun et al., CVPR'26", + "url": "https://github.com/SMSD75/SynthScope", + "evaluation": "training-free", + "metrics": { + "acc": 56.89, + "map": 32.05, + "ap": null, + "auroc": 82.59 + }, + "per_generator": { + "Crafter": { + "auroc": 87.91, + "ap": 53.66 + }, + "Gen2": { + "auroc": 90.0, + "ap": 58.91 + }, + "HotShot": { + "auroc": 76.97, + "ap": 21.25 + }, + "Lavie": { + "auroc": 86.42, + "ap": 53.53 + }, + "ModelScope": { + "auroc": 78.21, + "ap": 20.92 + }, + "MoonValley": { + "auroc": 82.67, + "ap": 22.83 + }, + "MorphStudio": { + "auroc": 83.22, + "ap": 28.34 + }, + "Show_1": { + "auroc": 82.29, + "ap": 30.09 + }, + "Sora": { + "auroc": 80.73, + "ap": 2.8 + }, + "WildScrape": { + "auroc": 77.48, + "ap": 28.22 + } + }, + "submitted_by": "SynthScope", + "date": "2026-11-02", + "verified": true, + "source": "repro/bench_results/stall_genvideo_full.json (benchmark/bench_eval_video.py --dataset genvideo, no --limit)", + "protocol_notes": "Training-free DINOv3 spatio-temporal likelihood under VATEX real-video statistics", + "synthscope_version": "0.1.0" +} diff --git a/conf/dataset/image/arforensics.yaml b/conf/dataset/image/arforensics.yaml index 995bb383dea580904ae4456f7f6c46577bbf6468..67fce083dc446855a3d943d08f09cbdfda6a5f4d 100644 --- a/conf/dataset/image/arforensics.yaml +++ b/conf/dataset/image/arforensics.yaml @@ -3,7 +3,8 @@ name: arforensics # Open-MAGVIT2), 6000 fakes + 6000 ImageNet reals each. (HF Yanran21/ARForensics, corrected test release, 21 GB, MIT) root_dir: /hddstore/ssalehi/SynthScope/ARForensics hf_repo_id: Yanran21/ARForensics -train_batch_size: 32 # D3QE's training data (train/val splits) is fetched only when a train_transform_list is set +with_train_splits: false # true: also build (and fetch) the train/val splits, D3QE's training data (39 GB) +train_batch_size: 32 val_batch_size: 64 test_batch_size: 512 num_workers: 8 diff --git a/conf/model/image/adapted_dino.yaml b/conf/model/image/adapted_dino.yaml new file mode 100644 index 0000000000000000000000000000000000000000..910f55db5c507c3106042d7d96c62f02adfc6934 --- /dev/null +++ b/conf/model/image/adapted_dino.yaml @@ -0,0 +1,15 @@ +defaults: + - /model/default + - _self_ + +# Adapter-tuned DINOv2: frozen DINOv2 ViT-L/14 with a trainable bottleneck adapter per block + a linear head. +model_type: adapted_dino +backbone_name: dinov2_vitl14 +use_adapter: true +bottleneck_dim: 64 +freeze_backbone: true + +# Test-time preprocessing: inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_256, center_crop_224, to_dtype, normalize_imagenet] +# Training augmentation. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype, normalize_imagenet] diff --git a/conf/model/image/aide.yaml b/conf/model/image/aide.yaml index d0e5233dd7a66b5c7257e4bbd942b9d90d267295..955788b47329cf0097da82510caa0b50ffd4ce85 100644 --- a/conf/model/image/aide.yaml +++ b/conf/model/image/aide.yaml @@ -19,3 +19,14 @@ dct: stride: 16 output_size: 256 grade_N: 6 + +# Test-time preprocessing (official): inference and benchmarks. +eval_transform_list: [aide_five_view] +# Training augmentation: the five native-resolution views, long side capped at 1024 (the recipe of the SynthScope_train +# checkpoint, repro/retrain_aide_native_v2.sbatch). +train_transform_list: [random_horizontal_flip, aide_five_view_train] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [aide_five_view_train] +# Caps the long side at 1024 to bound dataloader-worker memory during the end-of-training test; inference and +# benchmarks use the uncapped eval list. +test_transform_list: [aide_five_view_train] diff --git a/conf/model/image/bfree.yaml b/conf/model/image/bfree.yaml index 235d092ba05bffe7d5070d012d5d0a6227966f30..b3391dd92832451483b5ffc51cdc18c17f5d0a24 100644 --- a/conf/model/image/bfree.yaml +++ b/conf/model/image/bfree.yaml @@ -9,3 +9,8 @@ model_type: bfree timm_arch: vit_base_patch14_reg4_dinov2.lvd142m crop_size: 504 pretrained_backbone: false + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [to_dtype, normalize_imagenet] +# Native resolution (variable size): an end-of-training test through the training loaders needs +# DatasetConfig.test_batch_size=1 (no training recipe exists). diff --git a/conf/model/image/clip.yaml b/conf/model/image/clip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b45b12f1a10cc9d2b480cc42f1b4ca792a50c884 --- /dev/null +++ b/conf/model/image/clip.yaml @@ -0,0 +1,15 @@ +defaults: + - /model/default + - _self_ + +# UnivFD (Ojha et al., CVPR'23): frozen CLIP ViT-L/14 image features + a linear real/fake head. +model_type: clip +backbone_name: CLIP:ViT-L/14 +use_adapter: false +bottleneck_dim: 64 +freeze_backbone: true + +# Test-time preprocessing: inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_256, center_crop_224, to_dtype, normalize_clip] +# Training augmentation. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype, normalize_clip] diff --git a/conf/model/image/cnn.yaml b/conf/model/image/cnn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe13859f48b5f41a28c86da87f19a3111b047c1a --- /dev/null +++ b/conf/model/image/cnn.yaml @@ -0,0 +1,12 @@ +defaults: + - /model/default + - _self_ + +# CNNDetection (Wang et al., CVPR'20): ImageNet ResNet-50 fine-tuned end to end as a real/fake classifier. +model_type: cnn +freeze_backbone: false + +# Test-time preprocessing: inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_256, center_crop_224, to_dtype, normalize_imagenet] +# Training augmentation. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype, normalize_imagenet] diff --git a/conf/model/image/co_spy.yaml b/conf/model/image/co_spy.yaml index ac06a255f2baf813e4d60cb74436e743a33b1f5f..0a7481cd64bda1fb76de7891f915e565b1f85111 100644 --- a/conf/model/image/co_spy.yaml +++ b/conf/model/image/co_spy.yaml @@ -19,3 +19,8 @@ feat_interp_alpha: 0.2 num_classes: 2 freeze_backbone: true + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_384, center_crop_384, to_dtype] +# Training augmentation. +train_transform_list: [resize_short_384, gaussian_blur, jpeg_compression, random_crop_384, random_horizontal_flip, to_dtype] diff --git a/conf/model/image/cure.yaml b/conf/model/image/cure.yaml index 08c15e3b588665bff22e50c3bbcf3562e664f12b..0e9741a5cd4f6073bae043549e4ce5dcb05fa27b 100644 --- a/conf/model/image/cure.yaml +++ b/conf/model/image/cure.yaml @@ -13,3 +13,6 @@ base_filename: PE-Core-L14-336.pt base_checkpoint: null # local PE-Core-L14-336.pt override (or env SYNTHSCOPE_PE_CORE_CKPT) load_base_backbone: true # false = build the architecture only (smoke tests / key checks) label_smoothing: 0.0 + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [cure_eval] diff --git a/conf/model/image/d3qe.yaml b/conf/model/image/d3qe.yaml index a4821be3285b23c4585e941d1a5cd46d9d839ec0..647bf67baa0e77f1a62923e6f3cff5ab1e5fe1d1 100644 --- a/conf/model/image/d3qe.yaml +++ b/conf/model/image/d3qe.yaml @@ -8,3 +8,10 @@ defaults: # vq_ds16_c2i.pt) are fetched from their own releases, as in the official code. model_type: d3qe vq_checkpoint: null # local vq_ds16_c2i.pt; null -> $SYNTHSCOPE_LLAMAGEN_VQ, else download from the Hugging Face Hub + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_256, center_crop_256, to_dtype] +# Training augmentation. +train_transform_list: [resize_short_256, blur_jpeg_cnndet, random_crop_256, random_horizontal_flip, to_dtype] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [resize_short_256, blur_jpeg_cnndet_val, center_crop_256, to_dtype] diff --git a/conf/model/image/dda.yaml b/conf/model/image/dda.yaml index c23079a9222a1625ff09b3ab4e186185a3d18c24..4edd059c6826fdd67f48ad1f73f2b4fef39cff40 100644 --- a/conf/model/image/dda.yaml +++ b/conf/model/image/dda.yaml @@ -12,3 +12,7 @@ lora_rank: 8 lora_alpha: 1.0 lora_targets: [attn.qkv, attn.proj, mlp.fc1, mlp.fc2] freeze_backbone: true + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [center_crop_336, to_dtype, normalize_clip] +# No training lists: DDA trains on dataset image/dda_train, which builds its own official paired transforms. diff --git a/conf/model/image/dinov2.yaml b/conf/model/image/dinov2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a79710bd40d2ab38864c907e6acb73d86422c46 --- /dev/null +++ b/conf/model/image/dinov2.yaml @@ -0,0 +1,15 @@ +defaults: + - /model/default + - _self_ + +# DINOv2 probe: frozen DINOv2 ViT-L/14 + a linear real/fake head. +model_type: dinov2 +backbone_name: dinov2_vitl14 +use_adapter: false +bottleneck_dim: 64 +freeze_backbone: true + +# Test-time preprocessing: inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_256, center_crop_224, to_dtype, normalize_imagenet] +# Training augmentation. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype, normalize_imagenet] diff --git a/conf/model/image/dinov3.yaml b/conf/model/image/dinov3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..629c9a2f910e54998bd58073478c40797e02da9a --- /dev/null +++ b/conf/model/image/dinov3.yaml @@ -0,0 +1,15 @@ +defaults: + - /model/default + - _self_ + +# DINOv3 probe: frozen DINOv3 ViT-L/16 (gated weights) + a linear real/fake head. +model_type: dinov3 +backbone_name: dinov3_vitl16 +use_adapter: false +bottleneck_dim: 64 +freeze_backbone: true + +# Test-time preprocessing: inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_short_256, center_crop_256, to_dtype, normalize_dinov3] +# Training augmentation. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_256, random_horizontal_flip, to_dtype, normalize_dinov3] diff --git a/conf/model/image/effort.yaml b/conf/model/image/effort.yaml index a9d017a6d65e6586a93d9928b67046265cf52d36..c6d7e929fd2625be739a6b33dbeb191f5e1fb02d 100644 --- a/conf/model/image/effort.yaml +++ b/conf/model/image/effort.yaml @@ -7,3 +7,6 @@ defaults: model_type: effort clip_name: openai/clip-vit-large-patch14 svd_rank: 1023 + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [effort_cv2_resize_224, to_dtype, normalize_clip] diff --git a/conf/model/image/fat_former.yaml b/conf/model/image/fat_former.yaml index 60f632f7cb2ebed247718fb00e11bf7281f644f9..575105e370f2f309454ab0535014a794eddabcd1 100644 --- a/conf/model/image/fat_former.yaml +++ b/conf/model/image/fat_former.yaml @@ -11,3 +11,12 @@ num_classes: 2 classnames: [real, fake] freeze_backbone: true force_fp32: true + +# Official FatFormer (utils/dataset.py) normalizes with ImageNet mean/std, not CLIP's, despite the CLIP backbone (the +# official-weights eval reproduces 98.39 acc / 99.70 mAP with it). +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_256, center_crop_224, to_dtype, normalize_imagenet] +# Training augmentation. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype, normalize_imagenet] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [resize_short_256, center_crop_224, to_dtype, normalize_imagenet] diff --git a/conf/model/image/moejepa.yaml b/conf/model/image/moejepa.yaml index d0d0799d607614e6a91f5c4db3f613afb9c0e0cf..22923a9cd06849a0af64569d00ca19708c76f8f6 100644 --- a/conf/model/image/moejepa.yaml +++ b/conf/model/image/moejepa.yaml @@ -13,3 +13,6 @@ num_experts: 6 top_k: 2 router_temperature: 0.5 out_classes: 3 + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [vjepa2_processor] diff --git a/conf/model/image/npr.yaml b/conf/model/image/npr.yaml index 8be41d60769934bd8788ea6276c5b783f2919dfa..83af445729e5626afa82c400140f0aeab733ebac 100644 --- a/conf/model/image/npr.yaml +++ b/conf/model/image/npr.yaml @@ -5,3 +5,10 @@ defaults: model_type: npr num_classes: 2 freeze_backbone: false + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_256, to_dtype, normalize_imagenet] +# Training augmentation. +train_transform_list: [resize_256, random_crop_224, random_horizontal_flip, to_dtype, normalize_imagenet] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [resize_256, center_crop_224, to_dtype, normalize_imagenet] diff --git a/conf/model/image/omniaid.yaml b/conf/model/image/omniaid.yaml index 0bd8e84aa0e3da4c5a8df6d2bb0afe94dafdbdcd..bcbfef3c59e3a3175256f0a94cda26f95974a37c 100644 --- a/conf/model/image/omniaid.yaml +++ b/conf/model/image/omniaid.yaml @@ -16,3 +16,6 @@ moe_lambda_gating_cls: 0.1 dropout: 0.0 gradient_checkpointing_enable: false is_hybrid: true + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_336, to_dtype, normalize_clip] diff --git a/conf/model/image/omniaid_genimage.yaml b/conf/model/image/omniaid_genimage.yaml index 9a773fb5ebecca0b1966845bf7a9b15f2e37405f..2ab70badb7f64618154672c0ea467e64172bf961 100644 --- a/conf/model/image/omniaid_genimage.yaml +++ b/conf/model/image/omniaid_genimage.yaml @@ -16,3 +16,6 @@ moe_lambda_gating_cls: 0.1 dropout: 0.0 gradient_checkpointing_enable: false is_hybrid: true + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_512, resize_336, to_dtype, normalize_clip] diff --git a/conf/model/image/omniaid_v1.yaml b/conf/model/image/omniaid_v1.yaml index 8b08fe3278398eab51e32715731c1bf9a3363b16..538e2f7a142ace98e4df930567d53d0e63c95fe2 100644 --- a/conf/model/image/omniaid_v1.yaml +++ b/conf/model/image/omniaid_v1.yaml @@ -16,3 +16,6 @@ moe_lambda_gating_cls: 0.1 dropout: 0.0 gradient_checkpointing_enable: false is_hybrid: true + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [resize_512, resize_336, to_dtype, normalize_clip] diff --git a/conf/model/image/pgc.yaml b/conf/model/image/pgc.yaml index 7988e6d9c008970a0d80e36761fddc85f3ef6eab..774393b29c0b560f231cbedc0c30f5323c9b5ce4 100644 --- a/conf/model/image/pgc.yaml +++ b/conf/model/image/pgc.yaml @@ -19,3 +19,8 @@ label_smoothing: 0.1 tau_rgb: 0.5 tau_res: 0.5 num_classes: 1 + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [center_crop_224, to_dtype, normalize_imagenet] +# Training augmentation. +train_transform_list: [random_horizontal_flip, color_jitter_pgc, pad_random_crop_224, to_dtype, normalize_imagenet] diff --git a/conf/model/image/safe.yaml b/conf/model/image/safe.yaml index fcd9577b14469bde9e80305f59b75c8d3284d84b..d3adc7148675bb66f63b2c5ff1494e0626cd8998 100644 --- a/conf/model/image/safe.yaml +++ b/conf/model/image/safe.yaml @@ -6,3 +6,8 @@ defaults: # it does NOT use backbone_name. model_type=safe is dispatched before the CLIP fallback. model_type: safe freeze_backbone: false + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [center_crop_256, to_dtype] +# Training augmentation. +train_transform_list: [random_crop_256_pad, random_horizontal_flip, random_rotation_180, color_jitter_safe, to_dtype, random_mask_safe] diff --git a/conf/model/image/spai.yaml b/conf/model/image/spai.yaml index aa73ea5a11854a406e836f7adf8c3c085e1f4e40..674db4c03fd4fb1e3509db9dda4ba348be9c6ab4 100644 --- a/conf/model/image/spai.yaml +++ b/conf/model/image/spai.yaml @@ -15,4 +15,14 @@ use_sca: true patch_size: 224 patch_stride: 224 num_heads: 12 -attn_embed_dim: 1536 \ No newline at end of file +attn_embed_dim: 1536 + +# Test-time preprocessing (official): inference and benchmarks. +eval_transform_list: [cap_long_side_1024, to_dtype] +# Training augmentation. SPAI takes [0, 1] images; ImageNet normalization is applied inside the model. +train_transform_list: [resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [resize_short_256, center_crop_224, to_dtype] +# Batchable stand-in for the end-of-training test (the native-resolution eval list varies in size); inference and +# benchmarks use the eval list. +test_transform_list: [resize_short_256, center_crop_224, to_dtype] diff --git a/conf/model/image/vib_net.yaml b/conf/model/image/vib_net.yaml index cfc373daf1cc1284d9f656f74b024b0b30069a6c..58c79e23478f87fa9c9866401bc681ea7bb29ce9 100644 --- a/conf/model/image/vib_net.yaml +++ b/conf/model/image/vib_net.yaml @@ -13,3 +13,10 @@ vib_latent_dim: 256 dropout: 0.5 std_shift: 5.0 freeze_backbone: true + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [center_crop_224, to_dtype, normalize_clip] +# Training augmentation. +train_transform_list: [blur_jpeg_cnndet, pad_random_crop_224, random_horizontal_flip, to_dtype, normalize_clip] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [blur_jpeg_cnndet_val, center_crop_224, to_dtype, normalize_clip] diff --git a/conf/model/image/vib_net_official.yaml b/conf/model/image/vib_net_official.yaml index 24469311939b98b44a5389a4dacbc8f1340f29e2..fc423481ba9e7019c545e88955525dd14316db04 100644 --- a/conf/model/image/vib_net_official.yaml +++ b/conf/model/image/vib_net_official.yaml @@ -17,3 +17,10 @@ std_floor: 0.0 clip_fp32: true # The paper's trade-off parameter is not reported; 1e-3 is the standard VIB value (Alemi et al.). vib_beta: 0.001 + +# Test-time preprocessing (official): inference, benchmarks and the end-of-training test. +eval_transform_list: [center_crop_224, to_dtype, normalize_clip] +# Training augmentation. +train_transform_list: [blur_jpeg_cnndet, pad_random_crop_224, random_horizontal_flip, to_dtype, normalize_clip] +# Validation during training (differs from the test-time preprocessing above). +val_transform_list: [blur_jpeg_cnndet_val, center_crop_224, to_dtype, normalize_clip] diff --git a/dataset.py b/dataset.py index cd0d5870abf2da4a8ad39ce801ac84979264ee2b..e48d08571bbf3cefa9c521c5e1bdd417d23fdee3 100644 --- a/dataset.py +++ b/dataset.py @@ -1,6 +1,7 @@ from torch.utils.data import Dataset from typing import Literal from pathlib import Path +import functools import hashlib import json import os @@ -22,6 +23,7 @@ import numpy as np import torch import torch.nn.functional as F from decord import VideoReader, cpu +from omegaconf import OmegaConf from torch.utils.data import DataLoader from transformations import AIGITransformations from torchvision.io import decode_image, ImageReadMode @@ -71,6 +73,75 @@ def _has_videos(directory: Path) -> bool: return directory.is_dir() and any(_iter_videos(directory)) +def _config_list(section, name): + """``section.`` as a list, or None when the section or the key is absent or null.""" + value = None if section is None else section.get(name, None) + return None if value is None else list(value) + + +DATASET_CONF_DIR = Path(__file__).resolve().parent / "conf" / "dataset" + + +@functools.lru_cache(maxsize=None) +def _dataset_yaml_lists(name: str) -> tuple[dict[str, list | None], ...]: + """The ``train`` / ``val`` / ``eval`` transform lists shipped in the dataset yaml files of dataset ``name`` (matched + by file stem or ``name:``; several files may share a name), i.e. what DatasetConfig holds unless overridden.""" + found = [] + for path in sorted(DATASET_CONF_DIR.rglob("*.yaml")): + cfg = OmegaConf.load(path) + if name in (path.stem, cfg.get("name", None)): + found.append({key: _config_list(cfg, f"{key}_transform_list") for key in ("train", "val", "eval")}) + return tuple(found) + + +def _overridden(data, key: str, value: list) -> bool: + """Whether DatasetConfig's ``_transform_list`` was overridden (differs from its dataset yaml). False when the + yaml file or the key cannot be found.""" + shipped = [lists[key] for lists in _dataset_yaml_lists(str(data.get("name", ""))) if lists[key] is not None] + return bool(shipped) and value not in shipped + + +def _first_list(*candidates): + """The first candidate list that is set (not None), else None. An explicit ``[]`` counts as set: an empty list in + the model config intentionally stops the fallback to the dataset config (that split gets no transform).""" + return next((c for c in candidates if c is not None), None) + + +def resolve_transforms(config) -> tuple[AIGITransformations | None, AIGITransformations | None, AIGITransformations | None]: + """The ``(train, val, test)`` transforms of a run. Each comes from ``ModelConfig`` (the detector's own preprocessing) + and falls back to ``DatasetConfig`` for models whose config sets none. Precedence (first list set wins): + + * train: model ``train_transform_list``, dataset ``train_transform_list`` + * val: model ``val_transform_list``, model ``eval_transform_list``, dataset ``val_transform_list``, dataset + ``eval_transform_list`` + * test: model ``test_transform_list`` (a batchable stand-in for native-resolution eval lists), model + ``eval_transform_list``, dataset ``eval_transform_list`` + + A DatasetConfig list that was overridden (differs from its dataset yaml, e.g. an old + ``DatasetConfig.eval_transform_list=`` override) but loses to a model list is reported. + An empty list gives None.""" + model = getattr(config, "ModelConfig", None) + data = getattr(config, "DatasetConfig", None) + model_train = _config_list(model, "train_transform_list") + model_val = _first_list(_config_list(model, "val_transform_list"), _config_list(model, "eval_transform_list")) + model_test = _first_list(_config_list(model, "test_transform_list"), _config_list(model, "eval_transform_list")) + data_lists = {key: _config_list(data, f"{key}_transform_list") for key in ("train", "val", "eval")} + for key, model_list in (("train", model_train), ("val", model_val), ("eval", model_test)): + if (data_lists[key] is not None and model_list is not None and data_lists[key] != model_list + and _overridden(data, key, data_lists[key])): + print(f"[transforms] DatasetConfig.{key}_transform_list is ignored: the model config's list is used " + f"({model_list})", flush=True) + train = _first_list(model_train, data_lists["train"]) + val = _first_list(model_val, data_lists["val"], data_lists["eval"]) + test = _first_list(model_test, data_lists["eval"]) + # Val and test convert the decoded uint8 tensors to PIL first, so they run the exact inference code (benchmarks + # and the web UI feed PIL; tensor resizes differ by 1-2/255, and some eval steps take PIL only). Train stays on + # tensors for speed. + return (AIGITransformations(train) if train else None, + AIGITransformations(val, pil_input=True) if val else None, + AIGITransformations(test, pil_input=True) if test else None) + + class _PILFrames: """The part of decord's ``VideoReader`` API that ``sample_video_frames`` uses, for animated images decord cannot open (GIFs: decord fails with "broken metadata" on them). Every frame is composited to RGB at the canvas size, and the @@ -1128,16 +1199,13 @@ class CNNDetectionDataset(Dataset): class CNNDetectionDatasetModule: def __init__(self, config): self.config = config - train_list = list(config.DatasetConfig.train_transform_list) - eval_list = list(config.DatasetConfig.eval_transform_list) - self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.train_transform, self.val_transform, self.test_transform = resolve_transforms(config) prefetch = getattr(config.DatasetConfig, "prefetch", False) classes = config.DatasetConfig.get("train_classes", None) # ProGAN classes for train/val (None: all 20) classes = list(classes) if classes else None self.train_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "train", self.train_transform, prefetch=prefetch, classes=classes) - self.val_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "val", self.eval_transform, prefetch=prefetch, classes=classes) - self.test_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch) + self.val_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "val", self.val_transform, prefetch=prefetch, classes=classes) + self.test_dataset = CNNDetectionDataset(config.DatasetConfig.root_dir, "test", self.test_transform, prefetch=prefetch) def train_dataloader(self): return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) @@ -1149,9 +1217,9 @@ class CNNDetectionDatasetModule: return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) -def _split_train_subset(dataset, per_class, val_per_class, eval_transform, seed=0): +def _split_train_subset(dataset, per_class, val_per_class, val_transform, seed=0): """Keep a seeded ``per_class`` images per label of ``dataset`` for training and return a disjoint ``val_per_class`` - per label as a validation copy with ``eval_transform``: for recipes trained on part of a train split, validated on + per label as a validation copy with ``val_transform``: for recipes trained on part of a train split, validated on held-out training images (GenImage's ``val`` alias is its test split, which must not select the model).""" rng = random.Random(seed) train_idx, val_idx = [], [] @@ -1167,30 +1235,27 @@ def _split_train_subset(dataset, per_class, val_per_class, eval_transform, seed= ds.images = [dataset.images[i] for i in keep] ds.labels = [dataset.labels[i] for i in keep] ds.folder_names = [dataset.folder_names[i] for i in keep] - val.transform = eval_transform + val.transform = val_transform return val class GenImageDatasetModule: def __init__(self, config): self.config = config - train_list = list(config.DatasetConfig.train_transform_list) - eval_list = list(config.DatasetConfig.eval_transform_list) - self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.train_transform, self.val_transform, self.test_transform = resolve_transforms(config) prefetch = getattr(config.DatasetConfig, "prefetch", False) source_url = config.DatasetConfig.get("source_url", GenImageDataset.google_drive_folder_url) GenImageDataset.google_drive_folder_url = source_url GenImageDataset.hf_repo_id = config.DatasetConfig.get("hf_repo_id", None) GenImageDataset.hf_subdir = config.DatasetConfig.get("hf_subdir", None) self.train_dataset = GenImageDataset(config.DatasetConfig.root_dir, "train", self.train_transform, prefetch=prefetch) - self.test_dataset = GenImageDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch) + self.test_dataset = GenImageDataset(config.DatasetConfig.root_dir, "test", self.test_transform, prefetch=prefetch) per_class = config.DatasetConfig.get("train_per_class", None) # e.g. VIB-Net: 16k real + 16k SD v1.4 if per_class: self.val_dataset = _split_train_subset(self.train_dataset, int(per_class), - int(config.DatasetConfig.get("val_per_class", 1000)), self.eval_transform) + int(config.DatasetConfig.get("val_per_class", 1000)), self.val_transform) else: - self.val_dataset = GenImageDataset(config.DatasetConfig.root_dir, "val", self.eval_transform, prefetch=prefetch) + self.val_dataset = GenImageDataset(config.DatasetConfig.root_dir, "val", self.val_transform, prefetch=prefetch) def train_dataloader(self): return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) @@ -1757,14 +1822,11 @@ class UniversialFakeDetectionDataset(Dataset): class AIGIBenchDatasetModule: def __init__(self, config): self.config = config - train_list = list(config.DatasetConfig.train_transform_list) - eval_list = list(config.DatasetConfig.eval_transform_list) - self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.train_transform, self.val_transform, self.test_transform = resolve_transforms(config) prefetch = getattr(config.DatasetConfig, "prefetch", False) self.train_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "train", config.DatasetConfig.setting, self.train_transform, prefetch=prefetch) - self.val_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "val", config.DatasetConfig.setting, self.eval_transform, prefetch=prefetch) - self.test_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "test", config.DatasetConfig.setting, self.eval_transform, prefetch=prefetch) + self.val_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "val", config.DatasetConfig.setting, self.val_transform, prefetch=prefetch) + self.test_dataset = AIGIBenchDataset(config.DatasetConfig.root_dir, "test", config.DatasetConfig.setting, self.test_transform, prefetch=prefetch) def train_dataloader(self): return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) @@ -1779,11 +1841,10 @@ class AIGIBenchDatasetModule: class DiTFakeDatasetModule: def __init__(self, config): self.config = config - eval_list = list(config.DatasetConfig.eval_transform_list) - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_transform = resolve_transforms(config)[2] prefetch = getattr(config.DatasetConfig, "prefetch", False) repo_id = config.DatasetConfig.get("repo_id", DiTFakeDataset.repo_id) - self.test_dataset = DiTFakeDataset(config.DatasetConfig.root_dir, "test", self.eval_transform, prefetch=prefetch, repo_id=repo_id) + self.test_dataset = DiTFakeDataset(config.DatasetConfig.root_dir, "test", self.test_transform, prefetch=prefetch, repo_id=repo_id) def test_dataloader(self): return DataLoader(self.test_dataset, batch_size=self.config.DatasetConfig.test_batch_size, shuffle=False, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) @@ -1792,14 +1853,13 @@ class DiTFakeDatasetModule: class FreqNetDetectionDatasetModule: def __init__(self, config): self.config = config - eval_list = list(config.DatasetConfig.eval_transform_list) - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_transform = resolve_transforms(config)[2] prefetch = getattr(config.DatasetConfig, "prefetch", False) source_url = config.DatasetConfig.get("source_url", FreqNetDetectionDataset.google_drive_folder_url) self.test_dataset = FreqNetDetectionDataset( config.DatasetConfig.root_dir, "test", - self.eval_transform, + self.test_transform, prefetch=prefetch, source_url=source_url, ) @@ -1817,14 +1877,13 @@ class FreqNetDetectionDatasetModule: class UniversialFakeDetectionDatasetModule: def __init__(self, config): self.config = config - eval_list = list(config.DatasetConfig.eval_transform_list) - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_transform = resolve_transforms(config)[2] prefetch = getattr(config.DatasetConfig, "prefetch", False) source_url = config.DatasetConfig.get("source_url", UniversialFakeDetectionDataset.source_url) self.test_dataset = UniversialFakeDetectionDataset( config.DatasetConfig.root_dir, "test", - self.eval_transform, + self.test_transform, prefetch=prefetch, source_url=source_url, ) @@ -1842,8 +1901,7 @@ class UniversialFakeDetectionDatasetModule: class CoSpyDatasetModule: def __init__(self, config): self.config = config - eval_list = list(config.DatasetConfig.eval_transform_list) - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_transform = resolve_transforms(config)[2] prefetch = getattr(config.DatasetConfig, "prefetch", False) source_datasets = config.DatasetConfig.get("source_datasets", None) generator_models = config.DatasetConfig.get("generator_models", None) @@ -1854,7 +1912,7 @@ class CoSpyDatasetModule: self.test_dataset = CoSpyDataset( config.DatasetConfig.root_dir, "test", - self.eval_transform, + self.test_transform, prefetch=prefetch, source_datasets=list(source_datasets) if source_datasets else None, generator_models=list(generator_models) if generator_models else None, @@ -2015,10 +2073,7 @@ class VideoFolderDatasetModule: def __init__(self, config): self.config = config cfg = config.DatasetConfig - train_list = list(cfg.get("train_transform_list", [])) - eval_list = list(cfg.get("eval_transform_list", [])) - self.train_transform = AIGITransformations(train_list) if len(train_list) > 0 else None - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.train_transform, self.val_transform, self.test_transform = resolve_transforms(config) prefetch = bool(cfg.get("prefetch", False)) common = dict( num_frames=cfg.get("num_frames", 16), @@ -2030,8 +2085,8 @@ class VideoFolderDatasetModule: resize_interp=cfg.get("resize_interp", "bilinear"), ) self.train_dataset = VideoFolderDataset(cfg.root_dir, "train", self.train_transform, prefetch=prefetch, **common) - self.val_dataset = VideoFolderDataset(cfg.root_dir, "val", self.eval_transform, prefetch=prefetch, **common) - self.test_dataset = VideoFolderDataset(cfg.root_dir, "test", self.eval_transform, prefetch=prefetch, **common) + self.val_dataset = VideoFolderDataset(cfg.root_dir, "val", self.val_transform, prefetch=prefetch, **common) + self.test_dataset = VideoFolderDataset(cfg.root_dir, "test", self.test_transform, prefetch=prefetch, **common) def train_dataloader(self): return DataLoader(self.train_dataset, batch_size=self.config.DatasetConfig.train_batch_size, shuffle=True, num_workers=self.config.DatasetConfig.num_workers, persistent_workers=self.config.DatasetConfig.persistent_workers, pin_memory=self.config.DatasetConfig.pin_memory) @@ -2148,10 +2203,9 @@ class _TestOnlyVideoModule: def __init__(self, config): self.config = config cfg = config.DatasetConfig - eval_list = list(cfg.get("eval_transform_list", [])) - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.test_transform = resolve_transforms(config)[2] self.test_dataset = self.DATASET_CLASS( - cfg.root_dir, "test", self.eval_transform, prefetch=bool(cfg.get("prefetch", False)), + cfg.root_dir, "test", self.test_transform, prefetch=bool(cfg.get("prefetch", False)), num_frames=cfg.get("num_frames", 16), fps=cfg.get("fps", None), window_seconds=cfg.get("window_seconds", 2.0), frame_size=cfg.get("frame_size", 224), hf_repo_id=cfg.get("hf_repo_id", self.DATASET_CLASS.hf_repo_id), @@ -2922,10 +2976,9 @@ class _TestOnlyImageModule: def __init__(self, config): self.config = config cfg = config.DatasetConfig - eval_list = list(cfg.get("eval_transform_list", [])) - self.eval_transform = AIGITransformations(eval_list) if len(eval_list) > 0 else None + self.train_transform, self.val_transform, self.test_transform = resolve_transforms(config) self.test_dataset = self.DATASET_CLASS( - cfg.root_dir, "test", self.eval_transform, prefetch=bool(cfg.get("prefetch", False))) + cfg.root_dir, "test", self.test_transform, prefetch=bool(cfg.get("prefetch", False))) def train_dataloader(self): return None @@ -2993,20 +3046,18 @@ class SIDSetDatasetModule(_TestOnlyImageModule): class ARForensicsDatasetModule(_TestOnlyImageModule): - """The test split for benchmarking; with a ``train_transform_list`` in the config also the train and val splits - (D3QE's training data), validated with ``val_transform_list`` (default: ``eval_transform_list``).""" + """The test split for benchmarking; with ``DatasetConfig.with_train_splits`` (default false) also the train and val + splits (D3QE's training data, 39 GB), with the run's train and val transforms (see ``resolve_transforms``).""" DATASET_CLASS = ARForensicsDataset def __init__(self, config): super().__init__(config) cfg = config.DatasetConfig - train_list = list(cfg.get("train_transform_list", [])) self.train_dataset = self.val_dataset = None - if train_list: + if bool(cfg.get("with_train_splits", False)): prefetch = bool(cfg.get("prefetch", False)) - val_list = list(cfg.get("val_transform_list", cfg.get("eval_transform_list", []))) - self.train_dataset = ARForensicsDataset(cfg.root_dir, "train", AIGITransformations(train_list), prefetch=prefetch) - self.val_dataset = ARForensicsDataset(cfg.root_dir, "val", AIGITransformations(val_list), prefetch=prefetch) + self.train_dataset = ARForensicsDataset(cfg.root_dir, "train", self.train_transform, prefetch=prefetch) + self.val_dataset = ARForensicsDataset(cfg.root_dir, "val", self.val_transform, prefetch=prefetch) def _loader(self, dataset, batch_size, shuffle): cfg = self.config.DatasetConfig diff --git a/docs/ADD_DETECTOR.md b/docs/ADD_DETECTOR.md index 3e52e63d6269e7a986b684eb5b9bf10117d45bbb..0e2ad033f161749a716679eb62056f1724a3d703 100644 --- a/docs/ADD_DETECTOR.md +++ b/docs/ADD_DETECTOR.md @@ -19,13 +19,28 @@ for the maintainers and stop. - Imports + helper classes at module top-level (repo rule: no nested defs/imports). - Add `models///__init__.py` and a `README.md` (see structure below). 2. **`conf/model/image/.yaml`** (or `video/`): `defaults: [/model/default, _self_]`, then `model_type: ` - plus any model knobs. Add `conf/dataset/...` / `conf/trainer/...` only if a new recipe is needed. + plus any model knobs, and the model's preprocessing as named transform lists: + - `eval_transform_list` (required for image models): the **official** test-time preprocessing, used for + inference, benchmarks and the end-of-training test; + - `train_transform_list`: the training augmentation; + - `val_transform_list` (optional): validation during training, when it differs from the eval list; + - `test_transform_list` (optional): replaces the eval list for the end-of-training test only, e.g. a batchable + stand-in for SPAI's native-resolution eval, or AIDE's five views with the long side capped at 1024 to bound + dataloader-worker memory; benchmarks and inference still use `eval_transform_list`. + + Each name is a step built by `AIGITransformations` in `transformations.py`; add new step names there. + The model's lists take precedence over the dataset's: the `train/eval/val_transform_list` of a + `conf/dataset/...` config are fallbacks, used only for models whose config sets none, and a warning is printed + when a dataset list is shadowed. Validation and test pipelines run on PIL images like inference, so an + end-of-training test with `eval_transform_list` preprocesses exactly as the benchmarks do. + Video models are out of scope for now: their preprocessing is the registry's video settings plus the dataset + lists. Add `conf/dataset/...` / `conf/trainer/...` only if a new recipe is needed. 3. **Dispatch** — add a branch in `trainer.py` `_build_model` (the `if model_type == "": return SynthScopeModel(config)` chain, ~line 69+). The interface lazy-imports by `model_type` in `interface/backend/engine.py` — mirror it if needed. 4. **Interface registry** — `interface/backend/registry.py` `_add(ModelSpec(...))`: - `key`, `name`, `paper`, `modality` (`image`|`video`), `overrides=["model@ModelConfig=image/"]`, - `predict_kind`: `ce` (logits→`softmax[:,1]`), `bce` (`sigmoid`), `bce_npr`, `vib`, or `video`, - `checkpoint=".pth"`, `ckpt_kind`: `synthscope` (our trained weights, strict load) **or** `official_` + a `remap=remap_` function (rename official keys → our module names) for the authors' released weights, - - `transform`: the eval preprocessing — match the **official** eval exactly (resize vs crop, norm vs none). Add a `tf_*` helper if needed. (Getting this wrong silently tanks results — see SAFE: a resize destroyed the high-freq signal.) + - `transform=model_transform("image/")`: builds the eval preprocessing from the model config's `eval_transform_list` — match the **official** eval exactly there (resize vs crop, norm vs none). (Getting this wrong silently tanks results — see SAFE: a resize destroyed the high-freq signal.) - `weights`: `"official"` | `"synthscope_trained"` | `"training_free"` (shown in the web UI badge). 5. **Per-model `README.md`** — mirror `models/image/pgc/README.md`: short method explanation + official-repo/paper links, the pipeline, "SynthScope adaptations", and a results table in the **Paper / SynthScope_official / SynthScope_train** schema. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 672a76296ff5f96ac9527d2a56f2d6a6b0fb079d..8c5a3f85a89ac719e1dcc695f8a99a039d5aab1c 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -222,7 +222,8 @@ the GPU). Models are downloaded once into the Hugging Face cache (tens of GB eac - **A detector**: follow [`docs/ADD_DETECTOR.md`](ADD_DETECTOR.md); then add it to the Leaderboard as in section 3. - **A dataset**: add `conf/dataset/image/.yaml` with the source (`hf_repo_id`, `source_url`, ...), make it follow the - layout in section 2, and add a profile to `bench_eval.py` if its grouping is new. + layout in section 2, and add a profile to `bench_eval.py` if its grouping is new. Its transform lists are fallbacks, + used only for models whose config (`conf/model/...`) sets no lists of its own. - **A generator**: add a function `call(image_png | None, prompt, size=None) -> png bytes` and one entry in `GENERATORS` in `interface/backend/generators.py`. `image_png=None` means "create from the prompt alone". Then run section 4 with `--generator `. diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md index ac4e7a7a98d15670545b7780b6c820bbd8fc9770..12c7914e1d6d3de148f94ae1b5e1f4c480c58eba 100644 --- a/docs/CAPABILITIES.md +++ b/docs/CAPABILITIES.md @@ -72,10 +72,12 @@ To wire it in: `conf/model//.yaml` (`model_type: `), a branc with groups `model/`, `dataset/`, `trainer/`. Multi-GPU via 🤗 `accelerate` (`accelerate_config.yaml`). - **`dataset.py`** — image/video dataset loaders; `_decode_rgb` feeds uint8 `[3,H,W]` tensors; `sample_video_frames(..., edge_crop=)` for video. -- **`transformations.py`** — composable named transforms / augmentations: `resize_224`, - `resize_short_{256,384,448}`, `center_crop_{224,256,384,448}`, `random_crop_*`, `pad_random_crop_224` +- **`transformations.py`** — composable named transforms / augmentations: `resize_{224,256,336,512}`, + `resize_short_{256,384,448}`, `center_crop_{224,256,336,384,448}`, `random_crop_*`, `pad_random_crop_224` (PGC), `gaussian_blur`, `jpeg_compression`, `random_horizontal/vertical_flip`, `random_rotation_180`, - `color_jitter_safe`, `to_tensor`/`to_dtype`. **Eval-transform fidelity is critical** — artifact/ + `color_jitter_safe`, `to_tensor`/`to_dtype`. Each model config names its pipelines (`eval_transform_list`, + `train_transform_list`, optional `val_transform_list` / `test_transform_list`); the web registry builds + `ModelSpec.transform` from `eval_transform_list`. **Eval-transform fidelity is critical** — artifact/ low-level detectors (SAFE/AIDE/SPAI/PGC/NPR) are wrecked by a wrong resize; match the official eval exactly (see `repro/investigate/pgc_gap_investigation.md` and the SAFE crop-not-resize lesson). - **`metrics.py`** — torchmetrics binary `Accuracy` (real/fake), `AveragePrecision` (AP), @@ -170,6 +172,6 @@ New detectors are integrated **on a `detector/` branch and pushed for revie 1. Confirm it passes both rules (public repo + strong results). Read `ADD_DETECTOR.md`. 2. `git checkout -b detector/`; add the 5 files (model, conf, trainer dispatch, registry ModelSpec, README) + smoke test. -3. **Match the official eval transform exactly** in the `ModelSpec` (resize vs crop, norm) — this is the #1 silent-failure source. +3. **Match the official eval transform exactly** in the model config's `eval_transform_list` (resize vs crop, norm) — this is the #1 silent-failure source. 4. Reproduce on the paper's benchmark with official weights via `bench_eval.py --ckpt`; fix `remap_` for key mismatches; record per-gen AP/AUROC in the Paper/SynthScope_official/SynthScope_train schema. 5. (Optional) add an interpretability map. Push the branch; do **not** merge to `main`. diff --git a/experiments/image/adapted_dinov2_vitl14.sh b/experiments/image/adapted_dinov2_vitl14.sh index 1939aad6a7b304f97160b7c2c5e229cb46c88bb9..6e120a37fce88d124e14d423ad7a1aafc9cbad38 100644 --- a/experiments/image/adapted_dinov2_vitl14.sh +++ b/experiments/image/adapted_dinov2_vitl14.sh @@ -6,9 +6,7 @@ export CUDA_VISIBLE_DEVICES=1,2,3,4 export OMP_NUM_THREADS=1 accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py \ + model@ModelConfig=image/adapted_dino \ ModelConfig.bottleneck_dim=64 \ - ModelConfig.model_type=dino \ ModelConfig.use_adapter=true \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_dinov2] \ ModelConfig.backbone_name=dinov2_vitl14 \ No newline at end of file diff --git a/experiments/image/aide.sh b/experiments/image/aide.sh index 30fd7ceb728abff28148f1c90dc235da22511a28..92c41c3b934e9b34d5d8a0782feab7d155a2eac9 100644 --- a/experiments/image/aide.sh +++ b/experiments/image/aide.sh @@ -5,15 +5,16 @@ export TORCH_HOME=/nvmestore/ssalehi/.cache/torch export CUDA_VISIBLE_DEVICES=1,2 export OMP_NUM_THREADS=1 +# Five native-resolution views per image (aide.yaml lists): batch 8 and 5 workers bound memory, as in the run behind +# the README's SynthScope_train checkpoint (repro/retrain_aide_native_v2.sbatch). /home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ trainer@TrainerConfig=image/aide \ model@ModelConfig=image/aide \ dataset@DatasetConfig=image/genimage \ - DatasetConfig.train_batch_size=16 \ + DatasetConfig.train_batch_size=8 \ + DatasetConfig.num_workers=5 \ DatasetConfig.val_batch_size=16 \ DatasetConfig.test_batch_size=16 \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_256,to_dtype] \ logger.name=aide \ logger.mode=online \ logger.tags=[SynthScope,AIDE,genimage] diff --git a/experiments/image/clip_vitl14.sh b/experiments/image/clip_vitl14.sh index 353aedd90d33758c8037d79b5c2e8776d78103f1..adde513d0dbcd1885e0b0e4e960cabe32854c1ba 100644 --- a/experiments/image/clip_vitl14.sh +++ b/experiments/image/clip_vitl14.sh @@ -9,9 +9,7 @@ export OMP_NUM_THREADS=1 # accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py \ + model@ModelConfig=image/clip \ ModelConfig.bottleneck_dim=64 \ - ModelConfig.model_type=clip \ ModelConfig.use_adapter=false \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_clip] \ ModelConfig.backbone_name=CLIP:ViT-L/14 \ No newline at end of file diff --git a/experiments/image/co_spy.sh b/experiments/image/co_spy.sh index 9b04c60169c5478fdf7f41a11661737e1a246925..0261190b3baa11bffae77259ab5f3228166526f5 100644 --- a/experiments/image/co_spy.sh +++ b/experiments/image/co_spy.sh @@ -11,8 +11,6 @@ export OMP_NUM_THREADS=1 DatasetConfig.train_batch_size=16 \ DatasetConfig.val_batch_size=16 \ DatasetConfig.test_batch_size=16 \ - DatasetConfig.train_transform_list=[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype] \ - DatasetConfig.eval_transform_list=[resize_short_384,center_crop_384,to_dtype] \ logger.name=co_spy \ logger.mode=online \ logger.tags=[SynthScope,CoSpy,genimage] diff --git a/experiments/image/d3qe.sh b/experiments/image/d3qe.sh index 06a6c4009b9108c5ad3cbd8851fca331a5199188..e1a8b6f8456f1e5293205d122d0068bbf3c3fbc0 100644 --- a/experiments/image/d3qe.sh +++ b/experiments/image/d3qe.sh @@ -22,11 +22,9 @@ SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/d3qe} model@ModelConfig=image/d3qe \ dataset@DatasetConfig=image/arforensics \ DatasetConfig.root_dir=$DATA \ + DatasetConfig.with_train_splits=true \ DatasetConfig.train_batch_size=32 \ DatasetConfig.val_batch_size=64 \ - "+DatasetConfig.train_transform_list=[resize_short_256,blur_jpeg_cnndet,random_crop_256,random_horizontal_flip,to_dtype]" \ - "+DatasetConfig.val_transform_list=[resize_short_256,blur_jpeg_cnndet_val,center_crop_256,to_dtype]" \ - "DatasetConfig.eval_transform_list=[resize_short_256,center_crop_256,to_dtype]" \ TrainerConfig.save_path=$SAVE \ "metrics=[real_accuracy,fake_accuracy]" \ logger.name=d3qe \ diff --git a/experiments/image/dinov2_vitl14.sh b/experiments/image/dinov2_vitl14.sh index 6092b5c2760f8280eeeaecf88997948f9d1a0bb2..c21a09d9b85bc96288c0820d1116a7185dfe5352 100644 --- a/experiments/image/dinov2_vitl14.sh +++ b/experiments/image/dinov2_vitl14.sh @@ -9,9 +9,7 @@ export OMP_NUM_THREADS=1 # accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py accelerate launch --config_file accelerate_config.yaml --num_processes 4 main.py \ + model@ModelConfig=image/dinov2 \ ModelConfig.bottleneck_dim=64 \ - ModelConfig.model_type=dinov2 \ ModelConfig.use_adapter=false \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_dinov2] \ ModelConfig.backbone_name=dinov2_vitl14 \ No newline at end of file diff --git a/experiments/image/dinov3_vitl14.sh b/experiments/image/dinov3_vitl14.sh index 7ead0385236aed6b142b951020e7c867e89cd84b..16cb6034e384c5b67b72601b94c35158bd50e01a 100644 --- a/experiments/image/dinov3_vitl14.sh +++ b/experiments/image/dinov3_vitl14.sh @@ -9,9 +9,7 @@ export OMP_NUM_THREADS=1 # accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ + model@ModelConfig=image/dinov3 \ ModelConfig.bottleneck_dim=64 \ - ModelConfig.model_type=dinov3 \ ModelConfig.use_adapter=false \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype,normalize_dinov3] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_256,to_dtype,normalize_dinov3] \ ModelConfig.backbone_name=dinov3_vitl16 \ No newline at end of file diff --git a/experiments/image/fat_former.sh b/experiments/image/fat_former.sh index 346b0e48f866ef2e6945d44184a1259bf5ac0bd3..d9c02fe9fdda2345c60574331bd41507218c2eec 100644 --- a/experiments/image/fat_former.sh +++ b/experiments/image/fat_former.sh @@ -5,10 +5,6 @@ export TORCH_HOME=/nvmestore/ssalehi/.cache/torch export CUDA_VISIBLE_DEVICES=1,2 export OMP_NUM_THREADS=1 -# Official FatFormer (Michel-liu/FatFormer, utils/dataset.py) normalizes with **ImageNet** -# mean/std -- NOT CLIP stats -- despite the CLIP ViT-L/14 backbone. `normalize_dinov2` is the -# ImageNet Normalize in this repo, so we use it here to match the official preprocessing (and the -# deployed official-weights eval, which reproduces 98.39 acc / 99.70 mAP with ImageNet norm). /home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ trainer@TrainerConfig=image/fat_former \ model@ModelConfig=image/fat_former \ @@ -16,8 +12,6 @@ export OMP_NUM_THREADS=1 DatasetConfig.train_batch_size=32 \ DatasetConfig.val_batch_size=32 \ DatasetConfig.test_batch_size=32 \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype,normalize_dinov2] \ logger.name=fat_former \ logger.mode=online \ logger.tags=[SynthScope,FatFormer,aigibench] diff --git a/experiments/image/npr.sh b/experiments/image/npr.sh index 2186799ed289eb040a56cbe322227c6d1afb3fd9..0a5e576fda681a9573c1bd5d9103cf0019d5981b 100644 --- a/experiments/image/npr.sh +++ b/experiments/image/npr.sh @@ -25,8 +25,6 @@ SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/npr} "+DatasetConfig.train_classes=[car,cat,chair,horse]" \ DatasetConfig.train_batch_size=32 \ DatasetConfig.val_batch_size=64 \ - "DatasetConfig.train_transform_list=[resize_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" \ - "DatasetConfig.eval_transform_list=[resize_256,center_crop_224,to_dtype,normalize_dinov2]" \ TrainerConfig.save_path=$SAVE \ "metrics=[real_accuracy,fake_accuracy]" \ logger.name=npr \ diff --git a/experiments/image/pgc.sh b/experiments/image/pgc.sh index 7f614de3bad478bf159c3c205d79da2c43049ab4..8ccf4be9b71ec5c22fcd30498a83c473206603e5 100644 --- a/experiments/image/pgc.sh +++ b/experiments/image/pgc.sh @@ -25,8 +25,6 @@ SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/pgc} DatasetConfig.root_dir=$DATA \ DatasetConfig.train_batch_size=32 \ DatasetConfig.val_batch_size=64 \ - "DatasetConfig.train_transform_list=[random_horizontal_flip,color_jitter_pgc,pad_random_crop_224,to_dtype,normalize_dinov2]" \ - "DatasetConfig.eval_transform_list=[center_crop_224,to_dtype,normalize_dinov2]" \ TrainerConfig.save_path=$SAVE \ "metrics=[real_accuracy,fake_accuracy]" \ logger.name=pgc \ diff --git a/experiments/image/safe.sh b/experiments/image/safe.sh index 92cbab81c1520427c270931a42b298d8ba03c52a..3d29827ac9dd9e2fff53ad6872299b49e46a1242 100644 --- a/experiments/image/safe.sh +++ b/experiments/image/safe.sh @@ -7,14 +7,12 @@ export OMP_NUM_THREADS=1 /home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ trainer@TrainerConfig=image/safe \ + model@ModelConfig=image/safe \ dataset@DatasetConfig=image/aigibench \ - ModelConfig.model_type=safe \ ModelConfig.freeze_backbone=false \ DatasetConfig.train_batch_size=64 \ DatasetConfig.val_batch_size=64 \ DatasetConfig.test_batch_size=64 \ - DatasetConfig.train_transform_list=[random_crop_256_pad,random_horizontal_flip,random_rotation_180,color_jitter_safe,to_dtype,random_mask_safe] \ - DatasetConfig.eval_transform_list=[center_crop_256,to_dtype] \ TrainerConfig.epochs=100 \ TrainerConfig.lr=0.0005 \ TrainerConfig.weight_decay=0.05 \ diff --git a/experiments/image/spai.sh b/experiments/image/spai.sh index 258edad86f9fdf25acb455113aa209467b51a913..71510dc7a89e80b66a8ed0ed7c5ed5bc31bb4f48 100644 --- a/experiments/image/spai.sh +++ b/experiments/image/spai.sh @@ -5,15 +5,12 @@ export TORCH_HOME=/nvmestore/ssalehi/.cache/torch export CUDA_VISIBLE_DEVICES=1,2 export OMP_NUM_THREADS=1 -# SPAI expects [0,1] images (to_dtype only); ImageNet normalization is applied inside the model. /home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 2 main.py \ trainer@TrainerConfig=image/spai \ model@ModelConfig=image/spai \ DatasetConfig.train_batch_size=32 \ DatasetConfig.val_batch_size=32 \ DatasetConfig.test_batch_size=32 \ - DatasetConfig.train_transform_list=[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype] \ - DatasetConfig.eval_transform_list=[resize_short_256,center_crop_224,to_dtype] \ logger.name=spai \ logger.mode=online \ logger.tags=[SynthScope,SPAI,genimage] diff --git a/experiments/image/vib_net.sh b/experiments/image/vib_net.sh index bb612f94f9b51572a8a6a31e9408e8a73ee9847b..865fac27863189c66a9fd5cdd521a417c22a301d 100644 --- a/experiments/image/vib_net.sh +++ b/experiments/image/vib_net.sh @@ -31,8 +31,6 @@ SAVE=${SAVE:-/nvmestore/ssalehi/SynthScope/models/vib_net} DatasetConfig.prefetch=false \ DatasetConfig.train_batch_size=256 \ DatasetConfig.val_batch_size=256 \ - "DatasetConfig.train_transform_list=[blur_jpeg_cnndet,pad_random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - "DatasetConfig.eval_transform_list=[blur_jpeg_cnndet_val,center_crop_224,to_dtype,normalize_clip]" \ TrainerConfig.save_path=$SAVE \ "metrics=[real_accuracy,fake_accuracy]" \ logger.name=vib_net \ diff --git a/experiments/video/d3.sh b/experiments/video/d3.sh index cb3c8e8b269fdc0ca4f7290c23710513916e6f0a..036d068e7ba76480532131d186829804f3e9ad9d 100644 --- a/experiments/video/d3.sh +++ b/experiments/video/d3.sh @@ -8,7 +8,7 @@ export OMP_NUM_THREADS=1 # D3 is training-free: a frozen CLIP ViT-B/16 extracts per-frame features, and the per-video # second-order std becomes the detection score. Frames are decoded to 224x224 in [0,1] and # normalized by the dataset transform pipeline (normalize_clip here, matched to the CLIP -# backbone -- switch to normalize_dinov2 if you set a DINOv2/ResNet backbone). AP/AUROC are the +# backbone -- switch to normalize_imagenet if you set a DINOv2/ResNet backbone). AP/AUROC are the # meaningful metrics. The video benchmark must be present under DatasetConfig.root_dir; # this is a template (data download is out of scope). /home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 main.py \ diff --git a/experiments/video/restrav.sh b/experiments/video/restrav.sh index 779a3afa71e30919da611654f8d8e7840bc939db..656d8579f52ea003c7da0636bd618546a67fcec9 100644 --- a/experiments/video/restrav.sh +++ b/experiments/video/restrav.sh @@ -10,7 +10,7 @@ export OMP_NUM_THREADS=1 # 224x224 over a CENTER 2 s window (24 frames, BICUBIC) and fed to DINOv2 as RAW [0,1] with # *no* ImageNet normalization -- this matches the official ReStraV repo verbatim # (dinov2_features.preprocess = frames/255 + BICUBIC resize, no Normalize) and the deployed -# serving spec (registry video_norm = identity). Do NOT add normalize_dinov2 here: the +# serving spec (registry video_norm = identity). Do NOT add normalize_imagenet here: the # detector is served raw [0,1], so normalizing only at train time creates a train/serve # mismatch that collapses cross-generator transfer. Official optimizer is plain Adam (no # weight decay), 10 epochs -- matched below. The video benchmark must be present under diff --git a/experiments/video/stall.sh b/experiments/video/stall.sh index cd5f0f90d6390e1974f20578dc3cd0cb175c6dd8..f1c19e905e720e66842e201d710a2174158547db 100644 --- a/experiments/video/stall.sh +++ b/experiments/video/stall.sh @@ -8,7 +8,7 @@ export OMP_NUM_THREADS=1 # STALL (Ben Hayun et al., CVPR'26) is training-free: a frozen DINOv3 ViT-L/16 extracts # per-frame features, which are scored against a VATEX real-video calibration via two whitened # isotropic-Gaussian likelihood branches (spatial + temporal). Frames are decoded square at 224 -# in [0,1] and normalized with ImageNet stats -- normalize_dinov2 here, because the official +# in [0,1] and normalized with ImageNet stats -- normalize_imagenet here, because the official # DINOv3 transform uses ImageNet mean/std (NOT normalize_dinov3). AP/AUROC are the meaningful # metrics. The video benchmark must be present under DatasetConfig.root_dir. # @@ -24,8 +24,8 @@ export DINO_V3_WEIGHTS=${DINO_V3_WEIGHTS:-/hddstore/ssalehi/SynthScope/dinov3_ch DatasetConfig.test_batch_size=8 \ DatasetConfig.num_frames=16 \ DatasetConfig.frame_size=224 \ - DatasetConfig.train_transform_list=[normalize_dinov2] \ - DatasetConfig.eval_transform_list=[normalize_dinov2] \ + DatasetConfig.train_transform_list=[normalize_imagenet] \ + DatasetConfig.eval_transform_list=[normalize_imagenet] \ metrics=[real_accuracy,fake_accuracy,AP,AUC] \ logger.name=stall \ logger.mode=online \ diff --git a/interface/backend/engine.py b/interface/backend/engine.py index 7eb3560a8d6a5e78e48fea6b7de2e695613a948f..5d2d5204dad23d5c2ac4366dc273815bca174212 100644 --- a/interface/backend/engine.py +++ b/interface/backend/engine.py @@ -13,7 +13,6 @@ from collections import OrderedDict import numpy as np import torch -import torchvision.transforms as T from PIL import Image from safetensors.torch import load_file as load_safetensors @@ -61,14 +60,12 @@ def _overlay_to_png_b64(sal: torch.Tensor, base: Image.Image, out_size: int = 28 def _geometric_base(spec, pil: Image.Image) -> Image.Image: - """The model's input region in RGB (apply only the Resize/CenterCrop steps of the eval - transform, skipping ToTensor/Normalize) so the overlay aligns with the saliency. Custom + """The model's input region in RGB (apply only the resize/center-crop steps of the eval + transform, skipping to_dtype/normalize) so the overlay aligns with the saliency. Custom transforms (AIDE/SPAI native-res) fall back to the native image.""" - import torchvision.transforms as T base = pil.convert("RGB") - for t in getattr(spec.transform, "transforms", []) or []: - if isinstance(t, (T.Resize, T.CenterCrop)): - base = t(base) + for t in spec.transform.geometric_steps(): + base = t(base) return base @@ -329,7 +326,10 @@ class Engine: elif kind == "aide_dct": # AIDE: native-res DCT per-window energy (patch selection) if pil is None: return None - sal = spec.transform.dct_energy_map(pil) # [nH,Wn] + five_view = next((t for t in spec.transform.transforms if hasattr(t, "dct_energy_map")), None) + if five_view is None: + return None + sal = five_view.dct_energy_map(pil) # [nH,Wn] elif kind == "spai_patch": # SPAI: per-patch spectral fake-evidence over the 224-grid sal = model.patch_fake_map(x) # [nH,nW] or None (five-crop/large grid) if sal is None: @@ -351,7 +351,7 @@ class Engine: t = x[0].detach().float().cpu() if t.dim() == 4: t = torch.cat(list(t), dim=2) - norm = next((s for s in getattr(spec.transform, "transforms", []) if isinstance(s, T.Normalize)), None) + norm = spec.transform.normalize_step() if norm is not None: t = t * torch.tensor(norm.std)[:, None, None] + torch.tensor(norm.mean)[:, None, None] elif spec.key == "cure": # CuRe maps pixels to (x - 127.5) / 127.5 diff --git a/interface/backend/registry.py b/interface/backend/registry.py index fbda45eb057851febf3e79ad2365718e4c67d15f..297812b23fe2c784e921ec7932a166e3723a5f93 100644 --- a/interface/backend/registry.py +++ b/interface/backend/registry.py @@ -2,8 +2,8 @@ For each detector we record: * how to compose its SynthScope config (Hydra overrides, exactly like ``main.py``), - * its evaluation preprocessing (matched to the training/eval recipe or the verified - official-weights eval script under repro/), + * its evaluation preprocessing (the ``eval_transform_list`` of its model config, matched to the + official eval script or the verified official-weights eval under repro/), * which checkpoint file backs it and how that checkpoint loads (strict SynthScope vs. an official-weights key remap), and * how to turn a forward pass into a fake-probability. @@ -13,91 +13,22 @@ Label convention (dataset.py:_get_label): index 0 = real, index 1 = fake. import functools import os from dataclasses import dataclass, field +from pathlib import Path from typing import Callable, Optional -import torchvision.transforms as T +from omegaconf import OmegaConf + +from transformations import IMAGENET_MEAN, IMAGENET_STD, AIGITransformations # IMAGENET_*: the video_norm below + +MODEL_CONF_DIR = Path(__file__).resolve().parents[2] / "conf" / "model" -# --------------------------------------------------------------------------- -# Normalization constants (mirror transformations.py). -# --------------------------------------------------------------------------- -CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073) -CLIP_STD = (0.26862954, 0.26130258, 0.27577711) -IMAGENET_MEAN = (0.485, 0.456, 0.406) # == normalize_dinov2 -IMAGENET_STD = (0.229, 0.224, 0.225) -DINOV3_MEAN = (0.430, 0.411, 0.296) -DINOV3_STD = (0.213, 0.156, 0.143) - - -def _norm(mean, std): - return T.Normalize(mean=list(mean), std=list(std)) - - -def tf_resize_crop(resize: int, crop: int, mean=None, std=None) -> T.Compose: - """Resize shorter side -> ``resize``, center-crop ``crop``, ToTensor, optional Normalize.""" - steps = [T.Resize(resize), T.CenterCrop(crop), T.ToTensor()] - if mean is not None: - steps.append(_norm(mean, std)) - return T.Compose(steps) - - -def tf_crop(crop: int, mean=None, std=None) -> T.Compose: - """Center-crop ``crop`` with NO resize, ToTensor, optional Normalize. Official SAFE - preprocessing (scripts/eval.sh: --input_size 256 --transform_mode 'crop', no normalization) — - a resize would alter the high-frequency signal SAFE keys on.""" - steps = [T.CenterCrop(crop), T.ToTensor()] - if mean is not None: - steps.append(_norm(mean, std)) - return T.Compose(steps) - - -# AIDE's official native-resolution 5-view preprocessing (DCT patch selection on the native-size -# image). Importing this is light (no open_clip at module level); the heavy net is built lazily. -from models.image.aide.official_arch import AIDEFiveView # noqa: E402 -aide_five_view = AIDEFiveView() - -# CuRe's official eval preprocessing (PNG->JPEG96, 390 crop, 336 resize, JPEG70, mean/std 0.5), in cv2/PIL. -from models.image.cure.preprocess import CuReTransform # noqa: E402 -# Effort's official eval (DeepfakeBench load_rgb): cv2 bicubic squash to 224, then CLIP normalization. -from models.image.effort.preprocess import Cv2SquareResize # noqa: E402 -# MoE-JEPA's official eval: the V-JEPA 2 video processor (short side 292, center crop 256, ImageNet norm) on one frame. -from models.image.moejepa.preprocess import VJEPA2ImageTransform # noqa: E402 - - -def _cap_long_side(im, max_side: int): - """Downscale a PIL image (keeping aspect, min side 224) only if its long side exceeds - ``max_side`` — otherwise return it untouched at native resolution.""" - w, h = im.size - if max(w, h) > max_side: - s = max_side / max(w, h) - im = im.resize((max(int(w * s), 224), max(int(h * s), 224))) - return im - - -def tf_native_cap(max_side: int = 1024, mean=None, std=None) -> T.Compose: - """Native/arbitrary-resolution preprocessing: keep the image at its NATIVE size, only - downscaling (keeping aspect, min side 224) when the long side exceeds ``max_side`` to bound - memory. ToTensor (raw [0,1]), optional Normalize. This is SPAI's official any-resolution - path — a fixed Resize/CenterCrop would destroy the native-res spectral statistics it learns. - NB: produces variable-size tensors, so a ``native_res=True`` spec must be scored ONE image at - a time (bs=1); the web ``predict_image`` already does this, and bench_eval forces bs=1.""" - steps = [T.Lambda(functools.partial(_cap_long_side, max_side=max_side)), T.ToTensor()] - if mean is not None: - steps.append(_norm(mean, std)) - return T.Compose(steps) - - -def tf_square_resize(resize: int, crop: Optional[int], mean=None, std=None) -> T.Compose: - """Square Resize((resize,resize)), optional center-crop, ToTensor, optional Normalize. - - Matches the official FatFormer/NPR/AIDE eval preprocessing (a square resize, not a - shorter-side resize).""" - steps = [T.Resize((resize, resize))] - if crop is not None: - steps.append(T.CenterCrop(crop)) - steps.append(T.ToTensor()) - if mean is not None: - steps.append(_norm(mean, std)) - return T.Compose(steps) + +@functools.lru_cache(maxsize=None) +def model_transform(config_name: str) -> AIGITransformations: + """The test-time preprocessing of model config ``conf/model/.yaml`` (its ``eval_transform_list``): + the detector's official evaluation transform, PIL image in, model input tensor out. Cached: specs share the + instance, so it must not be mutated.""" + return AIGITransformations(list(OmegaConf.load(MODEL_CONF_DIR / f"{config_name}.yaml").eval_transform_list)) # --------------------------------------------------------------------------- @@ -155,15 +86,6 @@ def remap_bfree(sd: dict) -> dict: return {(k if k.startswith("patch_embed.") else "model." + k): v for k, v in sd.items()} -def tf_native(mean=None, std=None) -> T.Compose: - """The image exactly at its native resolution: ToTensor, optional Normalize, nothing else. Variable-size tensors, - so the spec must set ``native_res=True`` (scored one image at a time).""" - steps = [T.ToTensor()] - if mean is not None: - steps.append(_norm(mean, std)) - return T.Compose(steps) - - def remap_effort(sd: dict) -> dict: """Official Effort checkpoint (saved from a DataParallel EffortDetector) -> SynthScopeEffortModel: drop ``module.``, as the official demo does; the remaining ``backbone.*`` / ``head.*`` names are identical.""" @@ -190,7 +112,7 @@ class ModelSpec: predict_kind: str # ce | bce | bce_npr | vib | video checkpoint: Optional[str] # filename inside CHECKPOINTS_DIR (None = no weights file) ckpt_kind: str # synthscope | official_fatformer | official_aide | official_npr | none - transform: Optional[T.Compose] = None # image preprocessing (None for pure-video models) + transform: Optional[AIGITransformations] = None # image preprocessing (None for pure-video models) # Video-clip sampling (used when modality == "video"). num_frames: int = 16 fps: Optional[float] = None @@ -232,44 +154,36 @@ def _add(spec: ModelSpec): _add(ModelSpec( key="clip", name="UnivFD / CLIP", paper="Ojha et al., CVPR'23", modality="image", predict_kind="ce", checkpoint="clip.pth", ckpt_kind="synthscope", - overrides=["ModelConfig.model_type=clip", "ModelConfig.backbone_name=CLIP:ViT-L/14", - "ModelConfig.use_adapter=false", "ModelConfig.bottleneck_dim=64", - "ModelConfig.freeze_backbone=true"], - transform=tf_resize_crop(256, 224, CLIP_MEAN, CLIP_STD), + overrides=["model@ModelConfig=image/clip"], + transform=model_transform("image/clip"), notes="Frozen CLIP ViT-L/14 features + linear real/fake head (trained on ProGAN).", )) _add(ModelSpec( key="cnn", name="CNNDetection (ResNet-50)", paper="Wang et al., CVPR'20", modality="image", predict_kind="ce", checkpoint="cnn.pth", ckpt_kind="synthscope", - overrides=["ModelConfig.model_type=cnn", "ModelConfig.freeze_backbone=false"], - transform=tf_resize_crop(256, 224, IMAGENET_MEAN, IMAGENET_STD), + overrides=["model@ModelConfig=image/cnn"], + transform=model_transform("image/cnn"), notes="ImageNet ResNet-50 fine-tuned as a real/fake classifier (trained on ProGAN).", )) _add(ModelSpec( key="dinov2", name="DINOv2 probe", paper="Oquab et al., 2023", modality="image", predict_kind="ce", checkpoint="dinov2.pth", ckpt_kind="synthscope", - overrides=["ModelConfig.model_type=dinov2", "ModelConfig.backbone_name=dinov2_vitl14", - "ModelConfig.use_adapter=false", "ModelConfig.bottleneck_dim=64", - "ModelConfig.freeze_backbone=true"], - transform=tf_resize_crop(256, 224, IMAGENET_MEAN, IMAGENET_STD), + overrides=["model@ModelConfig=image/dinov2"], + transform=model_transform("image/dinov2"), notes="Frozen DINOv2 ViT-L/14 + linear head (trained on ProGAN).", )) _add(ModelSpec( key="dinov3", name="DINOv3 probe", paper="Meta AI, 2025", modality="image", predict_kind="ce", checkpoint="dinov3.pth", ckpt_kind="synthscope", - overrides=["ModelConfig.model_type=dinov3", "ModelConfig.backbone_name=dinov3_vitl16", - "ModelConfig.use_adapter=false", "ModelConfig.bottleneck_dim=64", - "ModelConfig.freeze_backbone=true"], - transform=tf_resize_crop(256, 256, DINOV3_MEAN, DINOV3_STD), + overrides=["model@ModelConfig=image/dinov3"], + transform=model_transform("image/dinov3"), notes="Frozen DINOv3 ViT-L/16 + linear head. Requires DINOv3 backbone weights.", )) _add(ModelSpec( key="adapted_dino", name="Adapter-tuned DINOv2", paper="SynthScope", modality="image", predict_kind="ce", checkpoint="adapted_dino.pth", ckpt_kind="synthscope", - overrides=["ModelConfig.model_type=adapted_dino", "ModelConfig.backbone_name=dinov2_vitl14", - "ModelConfig.use_adapter=true", "ModelConfig.bottleneck_dim=64", - "ModelConfig.freeze_backbone=true"], - transform=tf_resize_crop(256, 224, IMAGENET_MEAN, IMAGENET_STD), + overrides=["model@ModelConfig=image/adapted_dino"], + transform=model_transform("image/adapted_dino"), notes="DINOv2 with per-block bottleneck adapters; backbone frozen.", )) _add(ModelSpec( @@ -277,7 +191,7 @@ _add(ModelSpec( modality="image", predict_kind="ce", checkpoint="safe.pth", ckpt_kind="synthscope", remap=None, weights="synthscope_trained", overrides=["model@ModelConfig=image/safe"], - transform=tf_crop(256), + transform=model_transform("image/safe"), interpret="safe_hf", # DWT HH-band high-frequency artifact map notes="High-frequency (DWT bior1.3 HH-band) artifact detector. Serves the SynthScope-TRAINED checkpoint " "(full 40-epoch SAFE crop recipe, safe_full_v2/model_38), which reproduces the paper " @@ -291,7 +205,7 @@ _add(ModelSpec( ckpt_kind="official_fatformer", remap=remap_fatformer, weights="official", # The released checkpoint has FAA at blocks [7,15,23] => num_vit_adapter=3 (not the yaml's 8). overrides=["model@ModelConfig=image/fat_former", "ModelConfig.num_vit_adapter=3"], - transform=tf_square_resize(256, 224, IMAGENET_MEAN, IMAGENET_STD), + transform=model_transform("image/fat_former"), notes="Frozen CLIP + forgery-aware adapters + learnable prompts. Official ProGAN weights.", )) _add(ModelSpec( @@ -301,7 +215,7 @@ _add(ModelSpec( # Genuine 5-view AIDE arch; the official checkpoint supplies the ConvNeXt trunk (build it random # via convnext_pretrained=null to skip the 4.8GB download, then load the ckpt strict). overrides=["model@ModelConfig=image/aide", "ModelConfig.convnext_pretrained=null"], - transform=aide_five_view, # NATIVE-res DCT 5-view stack [5,3,256,256] (official preprocessing) + transform=model_transform("image/aide"), # NATIVE-res DCT 5-view stack [5,3,256,256] (official preprocessing) interpret="aide_dct", # native-res per-window DCT energy = the patch-selection saliency notes="ConvNeXt-XXL semantic + DCT/SRM low-level cues. The DCT patch selector runs at NATIVE " "resolution in preprocessing (resizing to 256 before the DCT costs ~12 mAP on the same " @@ -312,7 +226,7 @@ _add(ModelSpec( modality="image", predict_kind="bce", checkpoint="spai.pth", ckpt_kind="synthscope", weights="official", # shipped spai.pth = official SPAI weights remapped into the SynthScope class overrides=["model@ModelConfig=image/spai"], - transform=tf_native_cap(1024), # NATIVE resolution (raw [0,1], cap long side 1024) — the faithful any-res path + transform=model_transform("image/spai"), # NATIVE resolution (raw [0,1], cap long side 1024) — the faithful any-res path native_res=True, # score one image at a time (variable patch count) interpret="spai_patch", # per-patch spectral fake-evidence over the native 224-grid notes="Spectral learning (MFM ViT-B/16) + Spectral Context Attention at NATIVE/arbitrary resolution: each " @@ -327,7 +241,7 @@ _add(ModelSpec( # vib_net.pth = trained in SynthScope with the reconstructed recipe (experiments/image/vib_net.sh, job 267846); # reproduces the paper (15 generators: 94.08 AP / 85.22 acc vs 95.53 / 85.83). Official model + test transform. overrides=["model@ModelConfig=image/vib_net_official"], - transform=tf_crop(224, CLIP_MEAN, CLIP_STD), # official main.py: CenterCrop(224), ToTensor, CLIP norm (no resize) + transform=model_transform("image/vib_net_official"), # official main.py: CenterCrop(224), ToTensor, CLIP norm (no resize) notes="Variational information bottleneck on frozen CLIP ViT-L/14 features, trained in SynthScope on GenImage " "SD v1.4 with a recipe reconstructed from the paper (reproduces its 15-generator benchmark). The central " "224x224 pixels are scored; one latent sample per image, as the official model.", @@ -337,7 +251,7 @@ _add(ModelSpec( modality="image", predict_kind="vib", checkpoint="vib_net_official.pth", ckpt_kind="official_vibnet", remap=remap_vibnet, weights="official", overrides=["model@ModelConfig=image/vib_net_official"], - transform=tf_crop(224, CLIP_MEAN, CLIP_STD), # official main.py: CenterCrop(224), ToTensor, CLIP norm (no resize) + transform=model_transform("image/vib_net_official"), # official main.py: CenterCrop(224), ToTensor, CLIP norm (no resize) notes="The official VIB-Net model (fp32 CLIP ViT-L/14, no feature BatchNorm; one latent sample per image as the " "official code) with the official test transform. Loads the released checkpoints or a SynthScope-trained " "one (experiments/image/vib_net.sh).", @@ -349,12 +263,11 @@ _add(ModelSpec( # bfree.pth = the official BFREE_dino2reg4/model_epoch_best.pth unchanged (grip.unina.it); full model, strict load. overrides=["model@ModelConfig=image/bfree"], native_res=True, # official main_bfree.py: one image at a time at its native resolution - transform=tf_native(IMAGENET_MEAN, IMAGENET_STD), # official get_list_norm('resnet'): ToTensor + ImageNet norm + transform=model_transform("image/bfree"), # official get_list_norm('resnet'): ToTensor + ImageNet norm notes="DINOv2 ViT-B/14 with registers, trained end to end on bias-free data (inpainted fakes that share content " "and format with their real images). Uses the OFFICIAL released checkpoint (GRIP license: nonprofit use). " "Five 504x504 crops of the native-resolution image are scored and averaged, so nothing is resized.", )) -effort_transform = T.Compose([Cv2SquareResize(224), T.ToTensor(), _norm(CLIP_MEAN, CLIP_STD)]) for _key, _name, _ckpt, _note in [ ("effort", "Effort", "effort.pth", "trained on GenImage SD v1.4 (the paper's GenImage table)"), ("effort_chameleon", "Effort (Chameleon)", "effort_chameleon.pth", "trained on Chameleon"), @@ -365,7 +278,7 @@ for _key, _name, _ckpt, _note in [ weights="official", # .pth = the official effort_clip_L14_trainOn_{sdv14,chameleon}.pth unchanged (Google Drive, CC BY-NC 4.0). overrides=["model@ModelConfig=image/effort"], - transform=effort_transform, + transform=model_transform("image/effort"), notes=f"Orthogonal subspace decomposition: CLIP ViT-L/14 fine-tuned only along the minor SVD direction of each " f"attention projection, keeping the pretrained principal subspace. Official checkpoint, {_note}. The " f"whole image is squashed to 224x224 (cv2 bicubic), as the official DeepfakeBench test loader.", @@ -377,7 +290,7 @@ _add(ModelSpec( # moejepa.safetensors = the official moejepa_sidset.safetensors (v1.0 GitHub release, MIT), unchanged; it holds only # the trained tensors, the frozen V-JEPA 2 ViT-L comes from HF facebook/vjepa2-vitl-fpc64-256. overrides=["model@ModelConfig=image/moejepa"], - transform=VJEPA2ImageTransform(), + transform=model_transform("image/moejepa"), notes="Frozen V-JEPA 2 ViT-L (the image as a static 4-frame clip) with a residual mixture of experts in its last " "three blocks, gated-attention MIL pooling and a BayarConv noise branch fused by a learned gate. Official " "checkpoint, trained on SID-Set (social-media images: real, fully synthetic, tampered); P(fake) = P(fully " @@ -387,7 +300,7 @@ _add(ModelSpec( key="co_spy", name="CO-SPY", paper="Cheng et al., CVPR'25", modality="image", predict_kind="bce", checkpoint="co_spy.pth", ckpt_kind="synthscope", overrides=["model@ModelConfig=image/co_spy"], - transform=tf_resize_crop(384, 384, None, None), # raw [0,1]; per-branch norm inside the model + transform=model_transform("image/co_spy"), # raw [0,1]; per-branch norm inside the model notes="SigLIP semantic + SD-VAE-residual artifact streams, feature fusion (trained on ProGAN).", )) _add(ModelSpec( @@ -395,7 +308,7 @@ _add(ModelSpec( modality="image", predict_kind="bce", checkpoint="pgc.pth", ckpt_kind="synthscope", weights="official", # official PGC ProGAN weights load remap-free (FS 96.25 / hard-gens 94.35 vs our from-scratch 75.87); deploy PGC_train_progan_ckpt.pth as pgc.pth overrides=["model@ModelConfig=image/pgc"], - transform=tf_crop(224, IMAGENET_MEAN, IMAGENET_STD), # official PGC eval = PadCenterCrop(224), NO resize + transform=model_transform("image/pgc"), # official PGC eval = PadCenterCrop(224), NO resize interpret="pgc_peak", # peak-guided residual fake-evidence map (per-location score) notes="LoRA-DINOv2 RGB + YCbCr-residual CNN, peak-guided calibration. Uses OFFICIAL released weights. " "Eval is center-crop-224 NO resize (official PadCenterCrop): the YCbCr residual is resolution-" @@ -407,7 +320,7 @@ _add(ModelSpec( modality="image", predict_kind="ce", checkpoint="cure.pth", ckpt_kind="synthscope", weights="official", # cure.pth = the official 25MB cure_adapter.pt unchanged; frozen PE-Core base fetched from HF overrides=["model@ModelConfig=image/cure"], - transform=CuReTransform(), # official: PNG->JPEG96, SmallestMaxSize(390)+CenterCrop+Resize(336), JPEG70, mean=std=0.5 + transform=model_transform("image/cure"), # official: PNG->JPEG96, SmallestMaxSize(390)+CenterCrop+Resize(336), JPEG70, mean=std=0.5 notes="Frozen Perception-Encoder Core L/14 @336 + Q/V LoRA + source-response subspace + linear head, " "trained by regressing real-generated mixing ratios on GenImage SD-v1.4. Uses the OFFICIAL released " "adapter. Eval preprocessing is part of the method (PNG sources get a JPEG q96 round-trip and every " @@ -419,7 +332,7 @@ _add(ModelSpec( # dda.pth = the official DDA_ckpt.pth unchanged (HF Junwei-Xi/Dual-Data-Alignment); it holds the full model, # so the DINOv2 weights are not fetched separately. overrides=["model@ModelConfig=image/dda", "ModelConfig.pretrained_backbone=false"], - transform=tf_crop(336, CLIP_MEAN, CLIP_STD), # official Inference/inference.py: CenterCrop(336), ToTensor, CLIP norm + transform=model_transform("image/dda"), # official Inference/inference.py: CenterCrop(336), ToTensor, CLIP norm notes="DINOv2 ViT-L/14 + LoRA (rank 8) + linear head, trained on DDA's pixel- and frequency-aligned " "VAE-reconstructed fakes. Uses the OFFICIAL released checkpoint. The official test transform is a 336 " "center crop with no resize (smaller images are zero-padded), so large images are judged on their center.", @@ -430,20 +343,19 @@ _add(ModelSpec( # d3qe.pth = the official model_epoch_best.pth unchanged (HF Yanran21/D3QE); the frozen CLIP ViT-L/14 and LlamaGen # VQ-16 are fetched from their own releases (the model's pretrained_prefixes), all other keys load strictly. overrides=["model@ModelConfig=image/d3qe"], - transform=tf_resize_crop(256, 256), # official test path: Resize(256) short side, CenterCrop(256), ToTensor, no norm + transform=model_transform("image/d3qe"), # official test path: Resize(256) short side, CenterCrop(256), ToTensor, no norm notes="Detector for autoregressive (token-based) image generators: the quantization error of a frozen LlamaGen VQ " "tokenizer, read by a small transformer, fused with frozen CLIP ViT-L/14 features. Trained on LlamaGen vs " "ImageNet only. Uses the OFFICIAL released checkpoint.", )) # OmniAID's official eval preprocessing: v2 resizes straight to 336 (data/datasets.py); the v1-era releases used a # double resize (512x512, then 336x336) - the commented-out pipeline that the official code says to use for v1 weights. -omniaid_v1_transform = T.Compose([T.Resize((512, 512)), T.Resize((336, 336)), T.ToTensor(), _norm(CLIP_MEAN, CLIP_STD)]) -for _key, _name, _ckpt, _tf, _note in [ - ("omniaid", "OmniAID", "omniaid.pth", tf_square_resize(336, None, CLIP_MEAN, CLIP_STD), +for _key, _name, _ckpt, _note in [ + ("omniaid", "OmniAID", "omniaid.pth", "the recommended v2 release, trained on Mirage-Train (the authors' modern in-the-wild dataset)"), - ("omniaid_v1", "OmniAID (v1)", "omniaid_v1.pth", omniaid_v1_transform, + ("omniaid_v1", "OmniAID (v1)", "omniaid_v1.pth", "the v1 release trained on Mirage-Train, the paper's OmniAID-Mirage"), - ("omniaid_genimage", "OmniAID (GenImage)", "omniaid_genimage.pth", omniaid_v1_transform, + ("omniaid_genimage", "OmniAID (GenImage)", "omniaid_genimage.pth", "the release trained on GenImage SD v1.4 only, for the paper's academic GenImage protocol"), ]: _add(ModelSpec( @@ -451,7 +363,7 @@ for _key, _name, _ckpt, _tf, _note in [ modality="image", predict_kind="ce", checkpoint=_ckpt, ckpt_kind="official_omniaid", weights="official", # .pth = the official ckpt/checkpoint_omniaid_*.pth unchanged (HF Yunncheng/OmniAID); full model, strict load. overrides=[f"model@ModelConfig=image/{_key}"], - transform=_tf, + transform=model_transform(f"image/{_key}"), notes="Hybrid mixture of experts on CLIP ViT-L/14@336: a router picks semantic experts (human, animal, object, " "scene, ...) and an always-on artifact expert sees every image. Uses the OFFICIAL checkpoint: " + _note + ".", )) @@ -460,7 +372,7 @@ _add(ModelSpec( modality="image", predict_kind="bce_npr", checkpoint="npr.pth", ckpt_kind="official_npr", remap=remap_npr, weights="official", overrides=["model@ModelConfig=image/npr"], - transform=tf_square_resize(256, None, IMAGENET_MEAN, IMAGENET_STD), + transform=model_transform("image/npr"), interpret="npr_residual", # neighboring-pixel residual map (up-sampling artifacts) notes="Neighboring-pixel-residual ResNet. Official ProGAN weights.", )) @@ -510,8 +422,9 @@ _add(ModelSpec( # --------------------------------------------------------------------------- -# What each detector does to the input before scoring, shown in the UI as a short label + explanation. These mirror -# the transforms above, which in turn mirror each paper's evaluation (changing them would break reproduction). +# What each detector does to the input before scoring, shown in the UI as a short label + explanation. These describe +# the eval_transform_list of each model config (conf/model/), which mirrors the paper's evaluation (changing it would +# break reproduction). # --------------------------------------------------------------------------- _CENTER = ("Resize + center-crop", "The shorter side is scaled to {r} px and the central {c}×{c} square is used. " "The aspect ratio is kept; anything outside the square is not seen by the detector.") diff --git a/interface/scripts/train_model.sh b/interface/scripts/train_model.sh index 957e71febff3f46ffe49115dd64235ac2d36bd04..768964b2c5e3cf254d6a50f0088694800373f3eb 100644 --- a/interface/scripts/train_model.sh +++ b/interface/scripts/train_model.sh @@ -38,83 +38,66 @@ export DINO_V3_WEIGHTS_DIR="${DINO_V3_WEIGHTS_DIR:-/hddstore/ssalehi/SynthScope/ export CUDA_VISIBLE_DEVICES="$GPUS" NPROC=$(awk -F',' '{print NF}' <<< "$GPUS") -# Common transform lists. +# Transform lists come from each model config (conf/model/image/.yaml); the overrides below are this +# script's recipe where it differs from that config. T_CLIP_TRAIN="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" T_CLIP_EVAL="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" -T_IMNET_TRAIN="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" -T_IMNET_EVAL="[resize_short_256,center_crop_224,to_dtype,normalize_dinov2]" -T_RAW_TRAIN="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype]" -T_RAW_EVAL="[resize_short_256,center_crop_224,to_dtype]" OV=() # Hydra overrides case "$MODEL" in clip) - OV=(ModelConfig.model_type=clip ModelConfig.backbone_name=CLIP:ViT-L/14 - ModelConfig.use_adapter=false ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true - dataset@DatasetConfig=image/cnndetection - DatasetConfig.train_transform_list=$T_CLIP_TRAIN DatasetConfig.eval_transform_list=$T_CLIP_EVAL) + OV=(model@ModelConfig=image/clip + dataset@DatasetConfig=image/cnndetection) EPOCHS=${EPOCHS_OVERRIDE:-2} ;; dinov2) - OV=(ModelConfig.model_type=dinov2 ModelConfig.backbone_name=dinov2_vitl14 - ModelConfig.use_adapter=false ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true - dataset@DatasetConfig=image/cnndetection - DatasetConfig.train_transform_list=$T_IMNET_TRAIN DatasetConfig.eval_transform_list=$T_IMNET_EVAL) + OV=(model@ModelConfig=image/dinov2 + dataset@DatasetConfig=image/cnndetection) EPOCHS=${EPOCHS_OVERRIDE:-2} ;; dinov3) # Frozen DINOv3 ViT-L/16 probe (weights at $DINO_V3_WEIGHTS_DIR). DINOv3 normalization + 256 crops. - OV=(ModelConfig.model_type=dinov3 ModelConfig.backbone_name=dinov3_vitl16 - ModelConfig.use_adapter=false ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true - dataset@DatasetConfig=image/cnndetection - DatasetConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype,normalize_dinov3]" - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_256,to_dtype,normalize_dinov3]") + OV=(model@ModelConfig=image/dinov3 + dataset@DatasetConfig=image/cnndetection) EPOCHS=${EPOCHS_OVERRIDE:-2} ;; pgc) # LoRA-DINOv2-Large + residual stream. Train on the ~120K ProGAN subset for turnaround; - # crop-only (no resize) preserves the quantization residual; normalize_dinov2 = ImageNet. + # crop-only (no resize) preserves the quantization residual. OV=(trainer@TrainerConfig=image/pgc model@ModelConfig=image/pgc dataset@DatasetConfig=image/cospy_subset DatasetConfig.train_batch_size=32 DatasetConfig.val_batch_size=64 DatasetConfig.test_batch_size=64 - DatasetConfig.train_transform_list="[random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" - DatasetConfig.eval_transform_list="[center_crop_224,to_dtype,normalize_dinov2]") + ModelConfig.train_transform_list="[random_crop_224,random_horizontal_flip,to_dtype,normalize_imagenet]") EPOCHS=${EPOCHS_OVERRIDE:-2} ;; adapted_dino) # Adapters at every block retain backbone activations for backprop -> small batch. - OV=(ModelConfig.model_type=adapted_dino ModelConfig.backbone_name=dinov2_vitl14 - ModelConfig.use_adapter=true ModelConfig.bottleneck_dim=64 ModelConfig.freeze_backbone=true + OV=(model@ModelConfig=image/adapted_dino dataset@DatasetConfig=image/cnndetection - DatasetConfig.train_batch_size=64 DatasetConfig.val_batch_size=256 DatasetConfig.test_batch_size=256 - DatasetConfig.train_transform_list=$T_IMNET_TRAIN DatasetConfig.eval_transform_list=$T_IMNET_EVAL) + DatasetConfig.train_batch_size=64 DatasetConfig.val_batch_size=256 DatasetConfig.test_batch_size=256) EPOCHS=${EPOCHS_OVERRIDE:-2} ;; cnn) # Full ResNet-50 fine-tune (backbone trains) -> moderate batch. - OV=(ModelConfig.model_type=cnn ModelConfig.freeze_backbone=false + OV=(model@ModelConfig=image/cnn dataset@DatasetConfig=image/cnndetection - DatasetConfig.train_batch_size=128 DatasetConfig.val_batch_size=512 DatasetConfig.test_batch_size=512 - DatasetConfig.train_transform_list=$T_IMNET_TRAIN DatasetConfig.eval_transform_list=$T_IMNET_EVAL) + DatasetConfig.train_batch_size=128 DatasetConfig.val_batch_size=512 DatasetConfig.test_batch_size=512) EPOCHS=${EPOCHS_OVERRIDE:-3} ;; vib_net) OV=(trainer@TrainerConfig=image/vib_net model@ModelConfig=image/vib_net dataset@DatasetConfig=image/genimage - DatasetConfig.train_transform_list=$T_CLIP_TRAIN DatasetConfig.eval_transform_list=$T_CLIP_EVAL) + ModelConfig.train_transform_list=$T_CLIP_TRAIN ModelConfig.val_transform_list=$T_CLIP_EVAL) EPOCHS=${EPOCHS_OVERRIDE:-3} ;; spai) OV=(trainer@TrainerConfig=image/spai model@ModelConfig=image/spai dataset@DatasetConfig=image/spai_ldm - DatasetConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" - DatasetConfig.eval_transform_list="[resize_short_448,center_crop_448,to_dtype]") # 448 raw [0,1] -> 2x2 grid + SCA + ModelConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" + ModelConfig.val_transform_list="[resize_short_448,center_crop_448,to_dtype]" + ModelConfig.test_transform_list="[resize_short_448,center_crop_448,to_dtype]") # 448 raw [0,1] -> 2x2 grid + SCA EPOCHS=${EPOCHS_OVERRIDE:-5} ;; co_spy) OV=(trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy - dataset@DatasetConfig=image/cospy_subset - DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" - DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]") + dataset@DatasetConfig=image/cospy_subset) EPOCHS=${EPOCHS_OVERRIDE:-3} ;; safe) # AIGIBench (the paper recipe) is not on this node; train SAFE from scratch on ProGAN instead. - OV=(trainer@TrainerConfig=image/safe ModelConfig.model_type=safe ModelConfig.freeze_backbone=false - dataset@DatasetConfig=image/cnndetection - DatasetConfig.train_transform_list="[random_crop_256_pad,random_horizontal_flip,random_rotation_180,color_jitter_safe,to_dtype,random_mask_safe]" - DatasetConfig.eval_transform_list="[center_crop_256,to_dtype]") + OV=(trainer@TrainerConfig=image/safe model@ModelConfig=image/safe + dataset@DatasetConfig=image/cnndetection) EPOCHS=${EPOCHS_OVERRIDE:-10} ;; restrav) OV=(trainer@TrainerConfig=video/restrav model@ModelConfig=video/restrav diff --git a/models/image/bfree/README.md b/models/image/bfree/README.md index 78b21b17175e368db5d351e4077fbd040a9b6c18..154bffbd91aeb60d5fb43856fa4fafe626a12a71 100644 --- a/models/image/bfree/README.md +++ b/models/image/bfree/README.md @@ -30,7 +30,7 @@ loaded strictly. | `official_arch.py` = the official `code/networks/wrapper5crops.py`, verbatim | The five-crop wrapper is the model. GRIP license (nonprofit use; notices kept). | | `SynthScopeBFreeModel` subclasses `Wrapper5crops` and restores `nn.Module.load_state_dict` | The official wrapper splits a checkpoint itself; SynthScope loads the standard state dict, and `registry.remap_bfree` prefixes the timm keys with `model.` (the patch embedding stays `patch_embed.*`). | | `forward` returns the logit squeezed to `[B]`; `compute_loss` = BCE | `predict_kind="bce"`: P(fake) = sigmoid(logit), so 0.5 is the official logit > 0. | -| registry `native_res=True`, `tf_native(ImageNet)` | The official evaluation feeds each image at its own size, one at a time; any resize would change what the five crops see. | +| registry `native_res=True`, `eval_transform_list: [to_dtype, normalize_imagenet]` | The official evaluation feeds each image at its own size, one at a time; any resize would change what the five crops see. | | `pretrained_backbone: false` | The released checkpoint holds the whole model; `true` starts from the DINOv2 weights for training. | ## Fidelity check (identical to the official model) diff --git a/models/image/dda/dda.py b/models/image/dda/dda.py index d67f880437d95cae3e43cbe6f8e6cfd4105d42a9..47cf885dcfd92fdbf25ea858efc44d6f2bda806d 100644 --- a/models/image/dda/dda.py +++ b/models/image/dda/dda.py @@ -13,7 +13,7 @@ frequency space), not the architecture. ``SynthScopeDDAModel`` keeps the officia ``Junwei-Xi/Dual-Data-Alignment``; full model, 307.5M parameters) loads key-for-key as ``dda.pth``. Input contract: ``forward`` takes the tensor produced by the official test transform (center-crop 336 with no -resize, ToTensor, CLIP mean/std), i.e. registry ``tf_crop(336, CLIP_MEAN, CLIP_STD)``. +resize, ToTensor, CLIP mean/std), i.e. ``eval_transform_list: [center_crop_336, to_dtype, normalize_clip]`` in conf/model/image/dda.yaml. Training: ``compute_loss`` is the official objective (``Training/networks/trainer.py``): 0.5 x BCE on the logit + 0.5 x a contrastive loss on the CLS features (pytorch_metric_learning ``ContrastiveLoss(pos_margin=0, neg_margin=1)``, diff --git a/models/image/pgc/README.md b/models/image/pgc/README.md index 5f24520048b2cde14b699b51b2edd4632d2ea041..c6f6c8bad3faf9b7a2fd166ca4a149fe9579dad1 100644 --- a/models/image/pgc/README.md +++ b/models/image/pgc/README.md @@ -47,7 +47,7 @@ The model is a pure `nn.Module` (`forward`/`compute_loss`) like every SynthScope adaptations vs the official code (both preserve the method): 1. **Residual is computed inside the model.** The official builds the 6-channel input in the dataset - transform; here the dataset feeds a normal 3-channel ImageNet-normalized image (`normalize_dinov2`) + transform; here the dataset feeds a normal 3-channel ImageNet-normalized image (`normalize_imagenet`) and `forward` de-normalizes to `[0,1]`, computes the YCbCr residual, and concatenates — so the dataset/transform pipeline stays standard. 2. **Backbone loads by Hugging Face name** (auto-download) instead of a local checkpoint path. diff --git a/models/image/safe/README.md b/models/image/safe/README.md index 5d8bf9e618bac86f9dbf538f3434b79125d67504..502cc56350ab8df520c98c24736463ed98655061 100644 --- a/models/image/safe/README.md +++ b/models/image/safe/README.md @@ -67,11 +67,11 @@ SAFE's main contribution is its image transformation recipe. The official traini | Random mask | ratio `(0.00, 0.75)`, patch size `16`, `p = 0.5` | `random_mask_safe` | | Normalization | None | None | -The current SAFE experiment launcher uses: +The model config (`conf/model/image/safe.yaml`) sets: -```bash -DatasetConfig.train_transform_list=[random_crop_256_pad,random_horizontal_flip,random_rotation_180,color_jitter_safe,to_dtype,random_mask_safe] -DatasetConfig.eval_transform_list=[center_crop_256,to_dtype] +```yaml +eval_transform_list: [center_crop_256, to_dtype] +train_transform_list: [random_crop_256_pad, random_horizontal_flip, random_rotation_180, color_jitter_safe, to_dtype, random_mask_safe] ``` This matches the official transform order closely. `to_dtype` plays the same role as `ToTensor()` in the official code: it converts the loaded image tensor to `float32` and scales values to `[0, 1]`. diff --git a/models/image/spai/README.md b/models/image/spai/README.md index fbb2877862eb8be30ff430e006a4ab561ff18ecf..cf999ac156f6f6098409661ef41a20f505c3c7fd 100644 --- a/models/image/spai/README.md +++ b/models/image/spai/README.md @@ -106,16 +106,16 @@ The trainer config follows the paper's regime: AdamW with `lr = 5e-4`, weight de A launcher is provided at: ```bash -experiments/spai.sh +experiments/image/spai.sh ``` -It uses the default dataset (`genimage`). **The transform lists end in `to_dtype` only and contain no `normalize_*` transform**, because SPAI normalizes internally; feeding pre-normalized images would corrupt the frequency split. Dataset-side augmentations are `resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype` (eval: `resize_short_256, center_crop_224, to_dtype`). +It uses the default dataset (`genimage`) and the transform lists of `conf/model/image/spai.yaml`. **The transform lists end in `to_dtype` only and contain no `normalize_*` transform**, because SPAI normalizes internally; feeding pre-normalized images would corrupt the frequency split. Training augmentation is `resize_short_256, gaussian_blur, jpeg_compression, random_crop_224, random_horizontal_flip, to_dtype`; validation and the end-of-training test use `resize_short_256, center_crop_224, to_dtype`, while inference and benchmarks use the native-resolution eval list `cap_long_side_1024, to_dtype`. ## Reproduced benchmark results All benchmarks the paper reports, with our numbers (verified, traced to `repro/bench_results/` + `repro/benchmark_specs/`). -> **Served checkpoint:** the web app / `synthscope.load("spai")` now serves the **official SPAI weights remapped into the SynthScope class, scored at NATIVE resolution** (registry `weights="official"`, `native_res=True`, `tf_native_cap(1024)`, `spai.pth → repro/ckpts/spai_official_synthscope.pth`). This is the faithful any-res path — a fixed 448→2×2 resize destroys the native-res spectral statistics. The from-scratch fixed-448 retrain (88.8) is superseded. +> **Served checkpoint:** the web app / `synthscope.load("spai")` now serves the **official SPAI weights remapped into the SynthScope class, scored at NATIVE resolution** (registry `weights="official"`, `native_res=True`, `eval_transform_list: [cap_long_side_1024, to_dtype]`, `spai.pth → repro/ckpts/spai_official_synthscope.pth`). This is the faithful any-res path — a fixed 448→2×2 resize destroys the native-res spectral statistics. The from-scratch fixed-448 retrain (88.8) is superseded. | Benchmark | Paper | SynthScope_official (served, native-res) | SynthScope_train (retrain, superseded) | Status | |---|---|---|---|---| diff --git a/repro/MASTER_RESULTS.jsonl b/repro/MASTER_RESULTS.jsonl index 848d8a88e89135a5766ac3a30352a113b5d2aca4..e2ac88414e755191d47d09845a57d12f6c613012 100644 --- a/repro/MASTER_RESULTS.jsonl +++ b/repro/MASTER_RESULTS.jsonl @@ -384,3 +384,5 @@ {"date": "2026-11-01T12:38:06", "tag": "moejepa_evalgen", "model": "moejepa", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/EvalGEN/test", "n_generators": 5, "means": {"acc": 70.85, "map": 92.77, "mauroc": 86.43}} {"date": "2026-11-01T13:02:29", "tag": "moejepa_realchain", "model": "moejepa", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RealChain/test", "n_generators": 7, "means": {"acc": 66.39, "map": 71.74, "mauroc": 73.22}} {"date": "2026-11-01T13:59:48", "tag": "moejepa_rrdataset", "model": "moejepa", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RRDataset/test", "n_generators": 3, "means": {"acc": 59.67, "map": 65.09, "mauroc": 64.17}} +{"date": "2026-11-02T18:52:50", "tag": "restrav_genvideo_full", "model": "restrav", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 30.01, "map": 33.6, "map_real": 88.78, "mauroc": 43.06}} +{"date": "2026-11-02T20:31:58", "tag": "stall_genvideo_full", "model": "stall", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 56.89, "map": 32.05, "map_real": 98.09, "mauroc": 82.59}} diff --git a/repro/_interp_aide_test.py b/repro/_interp_aide_test.py index 90896d3156c388f7a2952fd2fef58adaf3cc21c4..d443db357965cbc6e01f178e9a01a2a125e783be 100644 --- a/repro/_interp_aide_test.py +++ b/repro/_interp_aide_test.py @@ -17,12 +17,12 @@ for label, p in [("fake", FAKE), ("real", REAL)]: if not os.path.exists(p): print(f" {label}: missing, skip"); continue pil = Image.open(p).convert("RGB") - # aide_dct branch uses spec.transform.dct_energy_map(pil); model is unused -> pass None. + # aide_dct branch uses the AIDEFiveView step's dct_energy_map(pil); model is unused -> pass None. amap = eng._interpret_map(None, spec, None, pil) if not amap: print(f" {label}: interpret_map=None (FAIL)"); ok = False; continue raw = base64.b64decode(amap.split(",", 1)[1]); png = Image.open(io.BytesIO(raw)); png.load() - em = spec.transform.dct_energy_map(pil) + em = spec.transform.transforms[0].dct_energy_map(pil) out = os.path.join(OUT, f"aide_{label}.png"); png.save(out) print(f" {label}: energy_map shape={tuple(em.shape)} -> PNG {png.size} {png.mode} {len(raw)}B -> {out}") print("ALL OK" if ok else "SOME FAILED") diff --git a/repro/bench_results/restrav_genvideo_full.json b/repro/bench_results/restrav_genvideo_full.json new file mode 100644 index 0000000000000000000000000000000000000000..2759503e4293eb8a2f87f716882315b860155298 --- /dev/null +++ b/repro/bench_results/restrav_genvideo_full.json @@ -0,0 +1,98 @@ +{ + "tag": "restrav_genvideo_full", + "model": "restrav", + "ckpt": "registry-default", + "benchmark": "video:pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", + "n_generators": 10, + "means": { + "acc": 30.01, + "map": 33.6, + "map_real": 88.78, + "mauroc": 43.06 + }, + "per_generator": { + "Crafter": { + "ap": 68.75, + "ap_real": 97.81, + "auroc": 89.15, + "real_acc": 0.88, + "fake_acc": 100.0, + "acc": 50.44 + }, + "Gen2": { + "ap": 6.37, + "ap_real": 71.5, + "auroc": 2.76, + "real_acc": 0.88, + "fake_acc": 36.23, + "acc": 18.56 + }, + "HotShot": { + "ap": 3.35, + "ap_real": 80.99, + "auroc": 0.1, + "real_acc": 0.88, + "fake_acc": 2.71, + "acc": 1.8 + }, + "Lavie": { + "ap": 25.86, + "ap_real": 77.99, + "auroc": 34.51, + "real_acc": 0.88, + "fake_acc": 48.86, + "acc": 24.87 + }, + "ModelScope": { + "ap": 59.16, + "ap_real": 96.96, + "auroc": 78.96, + "real_acc": 0.88, + "fake_acc": 99.29, + "acc": 50.08 + }, + "MoonValley": { + "ap": 3.02, + "ap_real": 82.75, + "auroc": 2.07, + "real_acc": 0.88, + "fake_acc": 30.83, + "acc": 15.86 + }, + "MorphStudio": { + "ap": 74.66, + "ap_real": 99.13, + "auroc": 92.58, + "real_acc": 0.88, + "fake_acc": 100.0, + "acc": 50.44 + }, + "Show_1": { + "ap": 66.49, + "ap_real": 99.11, + "auroc": 91.56, + "real_acc": 0.88, + "fake_acc": 100.0, + "acc": 50.44 + }, + "Sora": { + "ap": 0.29, + "ap_real": 97.16, + "auroc": 0.26, + "real_acc": 0.88, + "fake_acc": 8.93, + "acc": 4.9 + }, + "WildScrape": { + "ap": 28.08, + "ap_real": 84.39, + "auroc": 38.7, + "real_acc": 0.88, + "fake_acc": 64.5, + "acc": 32.69 + } + }, + "date": "2026-11-02T18:52:50", + "synthscope_version": "video" +} diff --git a/repro/bench_results/stall_genvideo_full.json b/repro/bench_results/stall_genvideo_full.json new file mode 100644 index 0000000000000000000000000000000000000000..da0d0ae7cf8e7194466d55509a6b05948c4b5c8c --- /dev/null +++ b/repro/bench_results/stall_genvideo_full.json @@ -0,0 +1,98 @@ +{ + "tag": "stall_genvideo_full", + "model": "stall", + "ckpt": "registry-default", + "benchmark": "video:pergen", + "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", + "n_generators": 10, + "means": { + "acc": 56.89, + "map": 32.05, + "map_real": 98.09, + "mauroc": 82.59 + }, + "per_generator": { + "Crafter": { + "ap": 53.66, + "ap_real": 97.83, + "auroc": 87.91, + "real_acc": 15.21, + "fake_acc": 99.07, + "acc": 57.14 + }, + "Gen2": { + "ap": 58.91, + "ap_real": 98.39, + "auroc": 90.0, + "real_acc": 15.21, + "fake_acc": 100.0, + "acc": 57.6 + }, + "HotShot": { + "ap": 21.25, + "ap_real": 97.58, + "auroc": 76.97, + "real_acc": 15.21, + "fake_acc": 97.43, + "acc": 56.32 + }, + "Lavie": { + "ap": 53.53, + "ap_real": 97.46, + "auroc": 86.42, + "real_acc": 15.21, + "fake_acc": 99.0, + "acc": 57.11 + }, + "ModelScope": { + "ap": 20.92, + "ap_real": 97.83, + "auroc": 78.21, + "real_acc": 15.21, + "fake_acc": 98.43, + "acc": 56.82 + }, + "MoonValley": { + "ap": 22.83, + "ap_real": 98.67, + "auroc": 82.67, + "real_acc": 15.21, + "fake_acc": 100.0, + "acc": 57.6 + }, + "MorphStudio": { + "ap": 28.34, + "ap_real": 98.45, + "auroc": 83.22, + "real_acc": 15.21, + "fake_acc": 99.14, + "acc": 57.18 + }, + "Show_1": { + "ap": 30.09, + "ap_real": 98.17, + "auroc": 82.29, + "real_acc": 15.21, + "fake_acc": 97.71, + "acc": 56.46 + }, + "Sora": { + "ap": 2.8, + "ap_real": 99.85, + "auroc": 80.73, + "real_acc": 15.21, + "fake_acc": 98.21, + "acc": 56.71 + }, + "WildScrape": { + "ap": 28.22, + "ap_real": 96.64, + "auroc": 77.48, + "real_acc": 15.21, + "fake_acc": 96.65, + "acc": 55.93 + } + }, + "date": "2026-11-02T20:31:58", + "synthscope_version": "video" +} diff --git a/repro/retrain_aide_cn025.sbatch b/repro/retrain_aide_cn025.sbatch index 357575e95e780c34a5b2bd87694b85a12388af00..14faa682c0509ae96094b95fb2952d6e2993b1f0 100644 --- a/repro/retrain_aide_cn025.sbatch +++ b/repro/retrain_aide_cn025.sbatch @@ -16,8 +16,9 @@ for i in $(seq 1 240); do [ -f /ivi/zfs/s0/original_homes/ssalehi/SynthScope/rep echo "node=$(hostname) scratch=$SCR data=$DATA/GenImage $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29651 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=5 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_retrain 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_aide_cn029.sbatch b/repro/retrain_aide_cn029.sbatch index 462bf9638d8e974c9313994d197920a0ec2d4b45..a395f7f58366829758364bad4bd544cd57d0d40a 100644 --- a/repro/retrain_aide_cn029.sbatch +++ b/repro/retrain_aide_cn029.sbatch @@ -15,8 +15,9 @@ for i in $(seq 1 240); do [ -f /ivi/zfs/s0/original_homes/ssalehi/SynthScope/rep echo "node=$(hostname) data=$DATA/GenImage $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29652 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=5 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_retrain 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_aide_cn031.sbatch b/repro/retrain_aide_cn031.sbatch index 128c9c0d9a233d515680b370f7cf00558b129015..48219bbeb76177712522d4d675296bc3d38d1dfb 100644 --- a/repro/retrain_aide_cn031.sbatch +++ b/repro/retrain_aide_cn031.sbatch @@ -10,8 +10,9 @@ export HF_HOME=$SC/huggingface TORCH_HOME=$SC/torch TRITON_CACHE_DIR=$SC/triton echo "node=$(hostname) scratch=$SCR data=$DATA/GenImage $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29631 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=32 DatasetConfig.val_batch_size=64 DatasetConfig.test_batch_size=64 DatasetConfig.num_workers=8 \ ++TrainerConfig.epochs=5 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_retrain 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_aide_faithful_mgpu.sbatch b/repro/retrain_aide_faithful_mgpu.sbatch index 750596eab81007d62bb79d35633bc2990a792288..52f7e8c364db8b3f620f5d3b8e5a6f787cbd46c9 100644 --- a/repro/retrain_aide_faithful_mgpu.sbatch +++ b/repro/retrain_aide_faithful_mgpu.sbatch @@ -20,8 +20,9 @@ $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file acc --num_processes 4 --main_process_port 29670 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/genimage \ DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_256,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_256,random_horizontal_flip,to_dtype]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_256,to_dtype]" \ + ModelConfig.test_transform_list="[resize_short_256,center_crop_256,to_dtype]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=12 \ ++TrainerConfig.epochs=5 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_faithful \ diff --git a/repro/retrain_aide_genimage_mgpu.sbatch b/repro/retrain_aide_genimage_mgpu.sbatch index bf00cf37b24a7942c1efe7f8806d3e74aa066bd8..5b1cf777595610013ada22bb470f21a78f635ca7 100644 --- a/repro/retrain_aide_genimage_mgpu.sbatch +++ b/repro/retrain_aide_genimage_mgpu.sbatch @@ -15,8 +15,9 @@ for i in $(seq 1 240); do [ -f repro/eval_logs/DATA_READY_cn029 ] && [ -d "$DATA echo "node=$(hostname) gpus=3 data=$DATA/GenImage real=$(ls $DATA/GenImage/train/real|wc -l) $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 3 --main_process_port 29653 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=6 \ ++TrainerConfig.epochs=5 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_genimage_mgpu 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_aide_native_v2.sbatch b/repro/retrain_aide_native_v2.sbatch index 5633beb96223a2742fa30a2781f85b911ed504db..db5cbe9a981aa2caca8797ec0728f66269e27b22 100644 --- a/repro/retrain_aide_native_v2.sbatch +++ b/repro/retrain_aide_native_v2.sbatch @@ -20,8 +20,9 @@ $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file acc --num_processes 4 --main_process_port 29673 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/genimage \ DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[random_horizontal_flip,aide_five_view_train]" \ - DatasetConfig.eval_transform_list="[aide_five_view_train]" \ + ModelConfig.train_transform_list="[random_horizontal_flip,aide_five_view_train]" \ + ModelConfig.val_transform_list="[aide_five_view_train]" \ + ModelConfig.test_transform_list="[aide_five_view_train]" \ DatasetConfig.train_batch_size=8 DatasetConfig.val_batch_size=16 DatasetConfig.test_batch_size=16 DatasetConfig.num_workers=5 \ ++TrainerConfig.epochs=10 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_native_v2 \ diff --git a/repro/retrain_aide_progan_mgpu.sbatch b/repro/retrain_aide_progan_mgpu.sbatch index d5af0cf9fbfaf99bb84a96f822ad5983cc65e0f4..ae8f8d594e3160bebf73360301e3f09da3be72b7 100644 --- a/repro/retrain_aide_progan_mgpu.sbatch +++ b/repro/retrain_aide_progan_mgpu.sbatch @@ -16,8 +16,9 @@ mkdir -p "$HF_HOME" "$TORCH_HOME" "$TRITON_CACHE_DIR" echo "node=$(hostname) gpus=3 data=$DATA/CNNDetection $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 3 --main_process_port 29654 \ main.py trainer@TrainerConfig=image/aide model@ModelConfig=image/aide dataset@DatasetConfig=image/cnndetection DatasetConfig.root_dir=$DATA/CNNDetection \ - DatasetConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=6 \ ++TrainerConfig.epochs=3 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=aide_progan_mgpu 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_cospy_cn025.sbatch b/repro/retrain_cospy_cn025.sbatch index c54c845541ffee9f682a5639d27b6152c129f42f..fe459138f00f0877e272bf92fbffe534ad241e1b 100644 --- a/repro/retrain_cospy_cn025.sbatch +++ b/repro/retrain_cospy_cn025.sbatch @@ -16,8 +16,9 @@ for i in $(seq 1 240); do [ -f /ivi/zfs/s0/original_homes/ssalehi/SynthScope/rep echo "node=$(hostname) scratch=$SCR data=$DATA/spai_train/cospy_subset $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29621 \ main.py trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy dataset@DatasetConfig=image/cospy_subset DatasetConfig.root_dir=$DATA/spai_train/cospy_subset \ - DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ + +ModelConfig.val_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + +ModelConfig.test_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=8 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=cospy_retrain 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_cospy_cn030.sbatch b/repro/retrain_cospy_cn030.sbatch index a5c40a73a811ddf5bcab4c6e5c79d1a13c2a9bb5..7ae0ba69c54ee34fdd085ec4b1876873f1bbc5b1 100644 --- a/repro/retrain_cospy_cn030.sbatch +++ b/repro/retrain_cospy_cn030.sbatch @@ -10,8 +10,9 @@ export HF_HOME=$SC/huggingface TORCH_HOME=$SC/torch TRITON_CACHE_DIR=$SC/triton echo "node=$(hostname) scratch=$SCR data=$DATA/spai_train/cospy_subset $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29621 \ main.py trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy dataset@DatasetConfig=image/cospy_subset DatasetConfig.root_dir=$DATA/spai_train/cospy_subset \ - DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ + +ModelConfig.val_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + +ModelConfig.test_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ DatasetConfig.train_batch_size=32 DatasetConfig.val_batch_size=64 DatasetConfig.test_batch_size=64 DatasetConfig.num_workers=8 \ ++TrainerConfig.epochs=8 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=cospy_retrain 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_cospy_full_mgpu.sbatch b/repro/retrain_cospy_full_mgpu.sbatch index 4236e44d93f5001b298b69a93c29a9e572511711..fbf6e88524560ca8d989ff17521a3eeaa375a1ba 100644 --- a/repro/retrain_cospy_full_mgpu.sbatch +++ b/repro/retrain_cospy_full_mgpu.sbatch @@ -18,8 +18,9 @@ $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file acc --num_processes 4 --main_process_port 29671 \ main.py trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy dataset@DatasetConfig=image/genimage \ DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ + +ModelConfig.val_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + +ModelConfig.test_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=12 \ ++TrainerConfig.epochs=4 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=cospy_full \ diff --git a/repro/retrain_cospy_genimage_gpu7.sh b/repro/retrain_cospy_genimage_gpu7.sh index 9fd242eebe48f352968ef7c88f2ae0368af6b618..7258bf5e8f02fe63ea726b60455b5f3ee1333cfd 100644 --- a/repro/retrain_cospy_genimage_gpu7.sh +++ b/repro/retrain_cospy_genimage_gpu7.sh @@ -15,8 +15,9 @@ echo "=== cospy genimage retrain start $(date) node=$(hostname) gpu=$CUDA_VISIBL --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29655 \ main.py trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy dataset@DatasetConfig=image/genimage \ DatasetConfig.root_dir=/hddstore/ssalehi/SynthScope/GenImage \ - DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ + +ModelConfig.val_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + +ModelConfig.test_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ DatasetConfig.train_batch_size=16 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 \ DatasetConfig.num_workers=8 DatasetConfig.prefetch=false \ ++TrainerConfig.epochs=3 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min \ diff --git a/repro/retrain_cospy_sub_gpu7.sh b/repro/retrain_cospy_sub_gpu7.sh index 086216fb8d729ae29f2af08c6b349dca1709dfef..3dbff1e22a5dfce32e61c2349952befceb0fbbc3 100644 --- a/repro/retrain_cospy_sub_gpu7.sh +++ b/repro/retrain_cospy_sub_gpu7.sh @@ -7,8 +7,9 @@ echo "=== cospy SUBSET retrain start $(date) ===" /home/ssalehi/miniconda3/bin/conda run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29655 \ main.py trainer@TrainerConfig=image/co_spy model@ModelConfig=image/co_spy dataset@DatasetConfig=image/genimage \ DatasetConfig.root_dir=/hddstore/ssalehi/SynthScope/GenImage_sub \ - DatasetConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_384,gaussian_blur,jpeg_compression,random_crop_384,random_horizontal_flip,to_dtype]" \ + +ModelConfig.val_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ + +ModelConfig.test_transform_list="[resize_short_384,center_crop_384,to_dtype]" \ DatasetConfig.train_batch_size=24 DatasetConfig.val_batch_size=32 DatasetConfig.test_batch_size=32 DatasetConfig.num_workers=8 DatasetConfig.prefetch=false \ ++TrainerConfig.epochs=5 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path="$SAVE" \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=cospy_sub 2>&1 | grep --line-buffered -vE "libpng|Testing:|Training:|Validation:" diff --git a/repro/retrain_pgc_long_cn029.sbatch b/repro/retrain_pgc_long_cn029.sbatch index 9a7df23745c3c4856831b2d3f9d898caf0784c4e..a9ac8facc51b357a4629aafb02f88fb01156bb4a 100644 --- a/repro/retrain_pgc_long_cn029.sbatch +++ b/repro/retrain_pgc_long_cn029.sbatch @@ -17,8 +17,9 @@ echo "node=$(hostname) data=$DATA/GenImage real=$(ls $DATA/GenImage/train/real 2 $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29663 \ main.py trainer@TrainerConfig=image/pgc model@ModelConfig=image/pgc ModelConfig.backbone_name=facebook/dinov2-large \ dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_dinov2]" \ + ModelConfig.train_transform_list="[resize_short_256,random_crop_224,random_horizontal_flip,to_dtype,normalize_dinov2]" \ + +ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_dinov2]" \ + +ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_dinov2]" \ DatasetConfig.train_batch_size=32 DatasetConfig.val_batch_size=64 DatasetConfig.test_batch_size=64 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=25 ++TrainerConfig.save=true ++TrainerConfig.save_policy=max ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=pgc_long 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_spai_cees.sbatch b/repro/retrain_spai_cees.sbatch index 66aa67e17f8ec05b300873d68bc90b0175916e6b..ab2363afd02fd84d0de33a8b39a4360163a6071e 100644 --- a/repro/retrain_spai_cees.sbatch +++ b/repro/retrain_spai_cees.sbatch @@ -16,8 +16,9 @@ RS=(rsync -aL --info=progress2 -e "ssh -o BatchMode=yes -o StrictHostKeyChecking echo "train reals: $(ls $DATA/spai_train/dmid_real/train/ldm/0_real 2>/dev/null|wc -l) | mfm: $(ls -la $SYNTHSCOPE_MFM_CKPT 2>/dev/null|awk '{print $5}')" "$CONDA" run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29661 \ main.py trainer@TrainerConfig=image/spai model@ModelConfig=image/spai dataset@DatasetConfig=image/spai_ldm DatasetConfig.root_dir=$DATA/spai_train/dmid_real \ - DatasetConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_448,center_crop_448,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" \ + ModelConfig.val_transform_list="[resize_short_448,center_crop_448,to_dtype]" \ + ModelConfig.test_transform_list="[resize_short_448,center_crop_448,to_dtype]" \ DatasetConfig.train_batch_size=8 DatasetConfig.val_batch_size=8 DatasetConfig.test_batch_size=8 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=10 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=spai_retrain_v2 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_spai_full_mgpu.sbatch b/repro/retrain_spai_full_mgpu.sbatch index 3472f025952a97a328e42b9d45c7f5d76d5054dc..1b97561d6176b7da91fdf6b0e86609a6065d73cf 100644 --- a/repro/retrain_spai_full_mgpu.sbatch +++ b/repro/retrain_spai_full_mgpu.sbatch @@ -13,8 +13,9 @@ RS=(rsync -aL -e "ssh -o BatchMode=yes -o StrictHostKeyChecking=no") echo "node=$(hostname) reals=$(ls $DATA/spai_train/dmid_real/train/ldm/0_real 2>/dev/null|wc -l) $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 4 --main_process_port 29672 \ main.py trainer@TrainerConfig=image/spai model@ModelConfig=image/spai dataset@DatasetConfig=image/spai_ldm DatasetConfig.root_dir=$DATA/spai_train/dmid_real \ - DatasetConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" \ - DatasetConfig.eval_transform_list="[resize_short_448,center_crop_448,to_dtype]" \ + ModelConfig.train_transform_list="[resize_short_448,gaussian_blur,jpeg_compression,random_crop_448,random_horizontal_flip,to_dtype]" \ + ModelConfig.val_transform_list="[resize_short_448,center_crop_448,to_dtype]" \ + ModelConfig.test_transform_list="[resize_short_448,center_crop_448,to_dtype]" \ DatasetConfig.train_batch_size=8 DatasetConfig.val_batch_size=8 DatasetConfig.test_batch_size=8 DatasetConfig.num_workers=12 \ ++TrainerConfig.epochs=8 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=spai_full 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_vib_cn025.sbatch b/repro/retrain_vib_cn025.sbatch index 784f464442c17fdea0d2732f2ea89939685994d7..5744e0d4cd88f94bcc378c07000b06ba412f9af5 100644 --- a/repro/retrain_vib_cn025.sbatch +++ b/repro/retrain_vib_cn025.sbatch @@ -24,8 +24,9 @@ echo "node=$(hostname) scratch=$SCR data=$DATA/GenImage flava=$HF_HOME genimage_ # --- train --- $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29641 \ main.py trainer@TrainerConfig=image/vib_net model@ModelConfig=image/vib_net dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + +ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=64 DatasetConfig.val_batch_size=128 DatasetConfig.test_batch_size=128 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=10 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=vib_retrain_flava 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/retrain_vib_cn029.sbatch b/repro/retrain_vib_cn029.sbatch index 7ef6dd1f0f28c3de995dd993f319e7a7bfaee4e9..8c4ad1b031d613385d421b54afe4c4c7f6af8970 100644 --- a/repro/retrain_vib_cn029.sbatch +++ b/repro/retrain_vib_cn029.sbatch @@ -19,8 +19,9 @@ echo "ssdstore" > /ivi/zfs/s0/original_homes/ssalehi/SynthScope/repro/eval_logs/ echo "node=$(hostname) data=$DATA/GenImage real=$(ls $DATA/GenImage/train/real 2>/dev/null|wc -l) $(date)" $CONDA run --no-capture-output -n SynthScope accelerate launch --config_file accelerate_config.yaml --num_processes 1 --main_process_port 29642 \ main.py trainer@TrainerConfig=image/vib_net model@ModelConfig=image/vib_net dataset@DatasetConfig=image/genimage DatasetConfig.root_dir=$DATA/GenImage \ - DatasetConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ - DatasetConfig.eval_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + ModelConfig.train_transform_list="[resize_short_256,gaussian_blur,jpeg_compression,random_crop_224,random_horizontal_flip,to_dtype,normalize_clip]" \ + ModelConfig.val_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ + +ModelConfig.test_transform_list="[resize_short_256,center_crop_224,to_dtype,normalize_clip]" \ DatasetConfig.train_batch_size=64 DatasetConfig.val_batch_size=128 DatasetConfig.test_batch_size=128 DatasetConfig.num_workers=10 \ ++TrainerConfig.epochs=10 ++TrainerConfig.save=true ++TrainerConfig.save_policy=min ++TrainerConfig.save_interval=1 ++TrainerConfig.save_path=$SAVE \ metrics=[real_accuracy,fake_accuracy,AP,AUC] logger.mode=offline logger.name=vib_retrain_flava 2>&1 | grep --line-buffered -vE "libpng (error|warning)" diff --git a/repro/train_pgc_progan_fix.sh b/repro/train_pgc_progan_fix.sh index f80788e70f7bf4968bccfff1f0b6ad7a8a3adf25..df5afabc2f8ace3b3788f292aede930482ce38a8 100644 --- a/repro/train_pgc_progan_fix.sh +++ b/repro/train_pgc_progan_fix.sh @@ -16,7 +16,8 @@ cd /ivi/zfs/s0/original_homes/ssalehi/SynthScope DatasetConfig.train_batch_size=128 \ DatasetConfig.val_batch_size=256 \ DatasetConfig.test_batch_size=256 \ - DatasetConfig.train_transform_list=[pad_random_crop_224,random_horizontal_flip,color_jitter_pgc,to_dtype,normalize_dinov2] \ - DatasetConfig.eval_transform_list=[center_crop_224,to_dtype,normalize_dinov2] \ + ModelConfig.train_transform_list=[pad_random_crop_224,random_horizontal_flip,color_jitter_pgc,to_dtype,normalize_dinov2] \ + +ModelConfig.val_transform_list=[center_crop_224,to_dtype,normalize_dinov2] \ + +ModelConfig.test_transform_list=[center_crop_224,to_dtype,normalize_dinov2] \ metrics=[real_accuracy,fake_accuracy,AP,AUC] \ logger.name=pgc_progan_fix logger.mode=offline diff --git a/tests/test_cospy_dataset.py b/tests/test_cospy_dataset.py index 52ec5a2b3de163ff0b798e9d233ae52167093864..cfb8b2cb35484a41eec837bdfad6fd6b0a0b65f9 100644 --- a/tests/test_cospy_dataset.py +++ b/tests/test_cospy_dataset.py @@ -3,7 +3,7 @@ Builds a fake staging tree that mimics the post-download CO-SPY-Bench layout, then exercises: -* Hydra composition of ``conf/dataset/cospy.yaml``. +* Hydra composition of ``conf/dataset/image/cospy.yaml``. * ``CoSpyDataset._restructure_to_cnndet_layout`` -- verifies the final on-disk tree is identical in shape to CNNDetection (``/ {0_real,1_fake}/*.png``), with hardlinked reals shared across all @@ -71,10 +71,10 @@ def _populate_staging_tree( def test_hydra_compose() -> None: - print("[1] Composing conf/dataset/cospy.yaml ...") + print("[1] Composing conf/dataset/image/cospy.yaml ...") cfg = compose_config( overrides=[ - "dataset@DatasetConfig=cospy", + "dataset@DatasetConfig=image/cospy", "DatasetConfig.root_dir=/tmp/__cospy_dummy__", "DatasetConfig.prefetch=false", ] @@ -171,7 +171,7 @@ def test_restructure_and_load() -> None: print("[6] Building CoSpyDatasetModule and iterating one batch ...") cfg = compose_config( overrides=[ - "dataset@DatasetConfig=cospy", + "dataset@DatasetConfig=image/cospy", f"DatasetConfig.root_dir={root}", "DatasetConfig.prefetch=false", "DatasetConfig.test_batch_size=4", diff --git a/tests/test_model_transforms.py b/tests/test_model_transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..121d28ff2ab883242d41128fdc13f5cad8552514 --- /dev/null +++ b/tests/test_model_transforms.py @@ -0,0 +1,316 @@ +"""Checks the preprocessing lists of the model configs (no downloads; MoE-JEPA's V-JEPA 2 processor runs only if cached). + +* every ``conf/model/image/*.yaml`` has a non-empty ``eval_transform_list``, and every ``*_transform_list`` of every model + and dataset config builds through ``AIGITransformations``; malformed names raise ``Invalid transform name``, +* each image detector's registry transform is its model config's ``eval_transform_list`` (one source of truth), +* the output shape of every eval pipeline on a 300x200 image, +* eval path == inference path: the test transform ``dataset.resolve_transforms`` builds, on ``dataset._decode_rgb`` + (the training loaders), equals the registry transform on ``bench_eval._open_rgb`` (benchmarks / web UI) exactly, on + PNG images of several sizes; models with a batchable ``test_transform_list`` compare their eval list through the + training-path wrapper instead, and their test list must batch across image sizes, +* pixel parity with hand-written torchvision v1 pipelines (the official eval code) for CLIP, NPR, PGC (incl. zero + padding), OmniAID v1, DDA and SAFE, +* a model's train / val / test outputs have the same per-sample rank, +* ``without_resize`` drops only resize / crop steps and refuses pipelines that resize inside, +* ``to_dtype`` equals ``to_tensor`` on a PIL image and ``ToDtype(float32, scale=True)`` on a uint8 tensor, exactly, +* ``resolve_transforms`` reports a DatasetConfig list shadowed by the model's only when it was overridden (differs from + the dataset yaml). +""" + +from __future__ import annotations + +import contextlib +import io +import os +import random +import sys +import tempfile +from pathlib import Path + +import numpy as np +import torch +import torchvision.transforms as T +import torchvision.transforms.functional as TF +import torchvision.transforms.v2 as v2 +from huggingface_hub import try_to_load_from_cache +from omegaconf import OmegaConf +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / "interface"), str(ROOT / "benchmark")] + +from transformations import (AIGITransformations, CLIP_MEAN, CLIP_STD, IMAGENET_MEAN, # noqa: E402 + IMAGENET_STD) +from backend.registry import list_specs, model_transform # noqa: E402 +from bench_eval import _open_rgb # noqa: E402 +from dataset import _decode_rgb, resolve_transforms # noqa: E402 +from models.image.cure.preprocess import SRC_FORMAT_KEY # noqa: E402 +from models.image.moejepa.moejepa import BACKBONE as VJEPA2 # noqa: E402 + +CONF = ROOT / "conf" +IMAGE_MODELS = sorted((CONF / "model" / "image").glob("*.yaml")) +TRAINING_KEYS = ("train_transform_list", "val_transform_list", "test_transform_list") +# (width, height) of the PNG test images: smaller than every crop, non-square both ways, square, long side > 1024. +SIZES = [(300, 200), (200, 150), (150, 260), (512, 512), (1300, 900)] +# Each image model's eval output on a 300x200 image (bfree / spai: native resolution). +EXPECTED_SHAPES = { + "adapted_dino": (3, 224, 224), "aide": (5, 3, 256, 256), "bfree": (3, 200, 300), "clip": (3, 224, 224), + "cnn": (3, 224, 224), "co_spy": (3, 384, 384), "cure": (3, 336, 336), "d3qe": (3, 256, 256), + "dda": (3, 336, 336), "dinov2": (3, 224, 224), "dinov3": (3, 256, 256), "effort": (3, 224, 224), + "fat_former": (3, 224, 224), "moejepa": (3, 256, 256), "npr": (3, 256, 256), "omniaid": (3, 336, 336), + "omniaid_genimage": (3, 336, 336), "omniaid_v1": (3, 336, 336), "pgc": (3, 224, 224), "safe": (3, 256, 256), + "spai": (3, 200, 300), "vib_net": (3, 224, 224), "vib_net_official": (3, 224, 224), +} +# The official eval code as torchvision v1 pipelines (PIL in). +GOLDEN = { + "clip": T.Compose([T.Resize(256), T.CenterCrop(224), T.ToTensor(), T.Normalize(CLIP_MEAN, CLIP_STD)]), + "npr": T.Compose([T.Resize((256, 256)), T.ToTensor(), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)]), + "pgc": T.Compose([T.CenterCrop(224), T.ToTensor(), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)]), + "omniaid_v1": T.Compose([T.Resize((512, 512)), T.Resize((336, 336)), T.ToTensor(), + T.Normalize(CLIP_MEAN, CLIP_STD)]), + "dda": T.Compose([T.CenterCrop(336), T.ToTensor(), T.Normalize(CLIP_MEAN, CLIP_STD)]), + "safe": T.Compose([T.CenterCrop(256), T.ToTensor()]), +} +# CuRe re-encodes PNG sources as JPEG q96 first. The training loaders decode to tensors and lose the source format, so +# their CuRe test path treats every image as non-PNG; the parity check compares against that branch of the inference. +FORMAT_DEPENDENT = {"cure"} + + +def _walk(node, prefix: str, found: dict[str, list[str]]) -> None: + if isinstance(node, dict): + for key, value in node.items(): + name = f"{prefix}{key}" + if str(key).endswith("_transform_list") and isinstance(value, list): + found[name] = [str(v) for v in value] + else: + _walk(value, name + ".", found) + + +def transform_lists(path: Path) -> dict[str, list[str]]: + """Every ``*_transform_list`` in a config file (any nesting depth), unresolved.""" + found = {} + _walk(OmegaConf.to_container(OmegaConf.load(path), resolve=False), "", found) + return found + + +def resolved(path: Path): + """``(train, val, test)`` of a run with model config ``path`` and a dataset config without transform lists.""" + return resolve_transforms(OmegaConf.create({"ModelConfig": OmegaConf.load(path), "DatasetConfig": {}})) + + +def noise_pil(size=(300, 200)) -> Image.Image: + arr = np.random.default_rng(0).integers(0, 256, (size[1], size[0], 3), dtype=np.uint8) + return Image.fromarray(arr, "RGB") + + +def noise_uint8(h=320, w=400) -> torch.Tensor: + return torch.randint(0, 256, (3, h, w), generator=torch.Generator().manual_seed(0), dtype=torch.uint8) + + +def write_pngs(directory: Path) -> list[Path]: + """Gradients plus noise at ``SIZES``, as PNG (lossless, so both decoders return the same pixels).""" + paths = [] + for i, (w, h) in enumerate(SIZES): + yy, xx = np.mgrid[0:h, 0:w] + base = np.stack([xx * 255 / w, yy * 255 / h, (xx + yy) * 127 / (w + h)], axis=-1) + arr = np.clip(base + np.random.default_rng(i).normal(0, 20, base.shape), 0, 255).astype(np.uint8) + paths.append(directory / f"img_{w}x{h}.png") + Image.fromarray(arr, "RGB").save(paths[-1]) + return paths + + +def vjepa2_cached() -> bool: + return isinstance(try_to_load_from_cache(VJEPA2, "video_preprocessor_config.json"), str) + + +def runnable(path: Path) -> bool: + if "vjepa2_processor" in transform_lists(path)["eval_transform_list"] and not vjepa2_cached(): + print(f" {path.stem}: skipped ({VJEPA2} processor not in the HF cache)") + return False + return True + + +def inference_transforms() -> dict[str, AIGITransformations]: + """Model config stem -> the transform the web UI / benchmarks run (the registry spec's, else ``model_transform``).""" + out = {path.stem: model_transform(f"image/{path.stem}") for path in IMAGE_MODELS} + for spec in list_specs(): + configs = [o.split("=", 1)[1] for o in spec.overrides if o.startswith("model@ModelConfig=image/")] + if configs: + out[configs[0].split("/", 1)[1]] = spec.transform + return out + + +def test_lists_build() -> None: + print("[A] every transform list builds; malformed names are rejected ...") + for path in IMAGE_MODELS: + assert len(transform_lists(path).get("eval_transform_list", [])) > 0, f"{path.name}: no eval_transform_list" + configs = sorted((CONF / "model").rglob("*.yaml")) + sorted((CONF / "dataset").rglob("*.yaml")) + n = 0 + for path in configs: + for names in transform_lists(path).values(): + AIGITransformations(names) + n += 1 + for name in ("resize_300", "resize_short_224", "center_crop_100", "resize_0256", "resize_", "center_crop_abc", + "resize_short_256x", "normalise_clip"): + try: + AIGITransformations([name]) + except ValueError as e: + assert str(e).startswith("Invalid transform name"), (name, e) + else: + raise AssertionError(f"{name!r} was accepted") + print(f" {n} lists in {len(configs)} configs ({len(IMAGE_MODELS)} image models, all with an eval list)") + + +def test_registry_matches_config() -> None: + print("[B] registry transform == the model config's eval_transform_list ...") + n = 0 + for spec in list_specs(): + configs = [o.split("=", 1)[1] for o in spec.overrides if o.startswith("model@ModelConfig=image/")] + if not configs: + continue + expected = transform_lists(CONF / "model" / f"{configs[0]}.yaml")["eval_transform_list"] + assert isinstance(spec.transform, AIGITransformations), (spec.key, type(spec.transform)) + assert list(spec.transform.transform_list) == expected, (spec.key, spec.transform.transform_list, expected) + n += 1 + print(f" {n} image specs") + + +def test_eval_shapes() -> None: + print("[C] eval pipelines on a 300x200 PIL image: float tensors of the expected shape ...") + assert set(EXPECTED_SHAPES) == {p.stem for p in IMAGE_MODELS}, "add the new model to EXPECTED_SHAPES" + pil = noise_pil() + for path in IMAGE_MODELS: + if not runnable(path): + continue + x = AIGITransformations(transform_lists(path)["eval_transform_list"])(pil.copy()) + assert isinstance(x, torch.Tensor) and x.is_floating_point() and torch.isfinite(x).all(), (path.stem, type(x)) + assert tuple(x.shape) == EXPECTED_SHAPES[path.stem], (path.stem, tuple(x.shape)) + print(f" {len(IMAGE_MODELS)} models") + + +def test_eval_path_is_inference_path() -> None: + print("[D] test transform on _decode_rgb == registry transform on _open_rgb (torch.equal) ...") + inference = inference_transforms() + with tempfile.TemporaryDirectory() as tmp: + pngs = write_pngs(Path(tmp)) + for path in IMAGE_MODELS: + if not runnable(path): + continue + lists = transform_lists(path) + test = resolved(path)[2] + if "test_transform_list" in lists: + assert test.transform_list == lists["test_transform_list"] and test.pil_input, (path.stem, test) + shapes = {tuple(test(_decode_rgb(p)).shape) for p in pngs} + assert len(shapes) == 1, (path.stem, "test_transform_list does not batch", shapes) + test = AIGITransformations(lists["eval_transform_list"], pil_input=True) + for p in pngs: + image = _open_rgb(p) + if path.stem in FORMAT_DEPENDENT: + del image.info[SRC_FORMAT_KEY] + got, want = test(_decode_rgb(p)), inference[path.stem](image) + assert got.shape == want.shape and torch.equal(got, want), \ + (path.stem, p.name, got.shape, want.shape, (got.float() - want.float()).abs().max().item()) + via = " (eval list via the training path)" if "test_transform_list" in lists else "" + print(f" {path.stem}: {len(pngs)} images equal{via}") + + +def test_golden() -> None: + print("[E] pixel parity with the official torchvision v1 pipelines ...") + with tempfile.TemporaryDirectory() as tmp: + pngs = write_pngs(Path(tmp)) + for stem, golden in GOLDEN.items(): + tf = model_transform(f"image/{stem}") + for p in pngs: + image = _open_rgb(p) + assert torch.equal(tf(image), golden(image)), (stem, p.name) + print(f" {', '.join(GOLDEN)}: {len(SIZES)} images each (incl. {SIZES[1][0]}x{SIZES[1][1]}: zero-padded crops)") + + +def test_split_ranks() -> None: + print("[F] train / val / test outputs of a model have the same per-sample rank ...") + random.seed(0) + torch.manual_seed(0) + with tempfile.TemporaryDirectory() as tmp: + image = _decode_rgb(write_pngs(Path(tmp))[-1]) + for path in IMAGE_MODELS: + if not runnable(path): + continue + ranks = {split: tf(image.clone()).ndim for split, tf in zip(("train", "val", "test"), resolved(path)) + if tf is not None} + assert len(set(ranks.values())) == 1, (path.stem, ranks) + print(f" {len(IMAGE_MODELS)} models") + + +def test_training_lists_on_tensor() -> None: + print("[G] train / val / test lists on a uint8 tensor (val / test through PIL, as resolve_transforms builds them) ...") + random.seed(0) + torch.manual_seed(0) + lists = [(path, key, names) for path in IMAGE_MODELS for key, names in transform_lists(path).items() + if key in TRAINING_KEYS] + lists += [(path, key, names) for path in sorted((CONF / "dataset" / "image").glob("*.yaml")) + for key, names in transform_lists(path).items()] + for path, key, names in lists: + x = AIGITransformations(names, pil_input=key != "train_transform_list")(noise_uint8()) + assert isinstance(x, torch.Tensor), (path.stem, key, type(x)) + if "to_dtype" in names: + assert x.is_floating_point() and torch.isfinite(x).all(), (path.stem, key, x.dtype) + print(f" {len(lists)} lists") + + +def test_without_resize() -> None: + print("[H] without_resize ...") + tf = AIGITransformations(["resize_short_256", "center_crop_224", "to_dtype", "normalize_clip"], pil_input=True) + native = tf.without_resize() + assert native.transform_list == ["to_dtype", "normalize_clip"] and native.pil_input, native + assert tuple(native(noise_pil()).shape) == (3, 200, 300) + for stem in ("spai", "cure", "effort", "aide", "moejepa"): + try: + model_transform(f"image/{stem}").without_resize() + except ValueError: + continue + raise AssertionError(f"{stem}: without_resize accepted a pipeline that resizes inside") + + +def test_to_dtype() -> None: + print("[I] to_dtype == to_tensor (PIL) / ToDtype(float32, scale=True) (uint8 tensor) ...") + to_dtype = AIGITransformations(["to_dtype"]) + pil = noise_pil() + assert torch.equal(to_dtype(pil), TF.to_tensor(pil)) + t = noise_uint8() + assert torch.equal(to_dtype(t), v2.ToDtype(torch.float32, scale=True)(t)) + + +def resolve_output(model: str, dataset: str, **data_overrides) -> str: + """What ``resolve_transforms`` prints for model config ``model`` on dataset config ``dataset`` with overrides.""" + data = OmegaConf.load(CONF / "dataset" / "image" / f"{dataset}.yaml") + for key, value in data_overrides.items(): + data[key] = value + out = io.StringIO() + with contextlib.redirect_stdout(out): + resolve_transforms(OmegaConf.create({"ModelConfig": OmegaConf.load(CONF / "model" / "image" / f"{model}.yaml"), + "DatasetConfig": data})) + return out.getvalue() + + +def test_shadow_warning() -> None: + print("[J] shadowed DatasetConfig lists are reported only when overridden ...") + assert resolve_output("npr", "genimage") == "" # the yaml's own lists: no report + changed = ["resize_short_256", "center_crop_224", "to_dtype"] + lines = resolve_output("npr", "genimage", eval_transform_list=changed).splitlines() + assert len(lines) == 1 and "DatasetConfig.eval_transform_list is ignored" in lines[0], lines + assert resolve_output("npr", "genimage", name="no_such_dataset", eval_transform_list=changed) == "" + + +if __name__ == "__main__": + os.chdir(ROOT) + test_lists_build() + test_registry_matches_config() + test_eval_shapes() + test_eval_path_is_inference_path() + test_golden() + test_split_ranks() + test_training_lists_on_tensor() + test_without_resize() + test_to_dtype() + test_shadow_warning() + print("Model transform checks passed") diff --git a/transformations.py b/transformations.py index 8748e3ebc783d49aebab434a9b0d08881ac45172..489fbe0a95a2b064bae8db6308ddee1ec8196a24 100644 --- a/transformations.py +++ b/transformations.py @@ -1,13 +1,33 @@ +import random +import re from io import BytesIO -from typing import Callable + import cv2 import numpy as np +import torch +import torchvision.transforms.functional as TF import torchvision.transforms.v2 as v2 from PIL import Image from scipy.ndimage import gaussian_filter from torchvision import tv_tensors -import torch -import random + +from models.image.aide.official_arch import AIDEFiveView +from models.image.cure.preprocess import CuReTransform +from models.image.effort.preprocess import Cv2SquareResize +from models.image.moejepa.preprocess import VJEPA2ImageTransform + +IMAGENET_MEAN, IMAGENET_STD = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] +CLIP_MEAN, CLIP_STD = [0.48145466, 0.4578275, 0.40821073], [0.26862954, 0.26130258, 0.27577711] +DINOV3_MEAN, DINOV3_STD = [0.430, 0.411, 0.296], [0.213, 0.156, 0.143] +SQUARE_RESIZES = (224, 256, 336, 512) # resize_: both sides to n (aspect not kept) +SHORT_RESIZES = (256, 384, 448) # resize_short_: shorter side to n +CROPS = (224, 256, 336, 384, 448) # center_crop_ (zero-padded when the image is smaller) +# The sized resize / crop names and their allowed sizes. +_SIZED = re.compile(r"(resize_short|resize|center_crop)_([1-9]\d*)") +_SIZES = {"resize_short": SHORT_RESIZES, "resize": SQUARE_RESIZES, "center_crop": CROPS} +# Name prefixes of steps that resize inside (whole official pipelines, SPAI's cap, aide_five_view*): an eval list holding +# one has no native-resolution form. +_RESIZING_STEPS = ("cap_long_side_1024", "effort_cv2_resize_224", "cure_eval", "vjepa2_processor", "aide_five_view") class CNNDetAugment: @@ -77,34 +97,70 @@ class RandomMask: mask[top:top + self.patch_size, left:left + self.patch_size] = 0 return tensor * mask.expand_as(tensor) + +class ToFloat: + """``to_dtype``: an image as a float tensor in [0, 1]. A PIL image goes through ``ToTensor`` (what the inference + transforms always used, so their pixels are unchanged); a uint8 tensor (the training loaders) through + ``ToDtype(float32, scale=True)``.""" + + def __init__(self): + self._tensor = v2.ToDtype(torch.float32, scale=True) + + def __call__(self, image): + return TF.to_tensor(image) if isinstance(image, Image.Image) else self._tensor(image) + + def __repr__(self): + return "ToFloat()" + + +class CapLongSide: + """Downscale a PIL image (keeping aspect, min side 224) only if its long side exceeds ``max_side``; otherwise + leave it at native resolution (SPAI's any-resolution path).""" + + def __init__(self, max_side: int): + self.max_side = max_side + + def __call__(self, im): + w, h = im.size + if max(w, h) > self.max_side: + s = self.max_side / max(w, h) + im = im.resize((max(int(w * s), 224), max(int(h * s), 224))) + return im + + def __repr__(self): + return f"CapLongSide({self.max_side})" + + class AIGITransformations: - def __init__(self, transform_list: list[Callable]): + """A named preprocessing pipeline. Each detector's lists live in its model config (``conf/model/.../.yaml``: + ``eval_transform_list``, ``train_transform_list``, optional ``val_transform_list``); the same names build the + inference transform (PIL in) and the training transforms (uint8 tensors in). + + With ``pil_input=True`` a uint8 ``[3, H, W]`` image tensor (what the training loaders decode) is converted to a PIL + image first, so the pipeline runs exactly the inference code; ``[T, C, H, W]`` video clips are left as they are.""" + + def __init__(self, transform_list: list[str], pil_input: bool = False): self.transform_list = transform_list + self.pil_input = pil_input self.transforms = [self._create_transform(transform) for transform in self.transform_list] - + def _create_transform(self, transform_name: str): - if transform_name == "resize_224": - return v2.Resize((224, 224)) - elif transform_name == "resize_256": # square, as NPR/FatFormer/CNNDetection-style training (Resize((256, 256))) - return v2.Resize((256, 256)) - elif transform_name == "resize_short_256": - return v2.Resize(256) - elif transform_name == "resize_short_384": - return v2.Resize(384) - elif transform_name == "resize_short_448": - return v2.Resize(448) - elif transform_name == "center_crop_224": - return v2.CenterCrop(224) - elif transform_name == "center_crop_256": - return v2.CenterCrop(256) - elif transform_name == "center_crop_384": - return v2.CenterCrop(384) - elif transform_name == "center_crop_448": - return v2.CenterCrop(448) + sized = _SIZED.fullmatch(transform_name) + if sized: + kind, n = sized.group(1), int(sized.group(2)) + if n not in _SIZES[kind]: + raise ValueError(f"Invalid transform name: {transform_name}") + if kind == "resize_short": + return v2.Resize(n) + if kind == "resize": # square, e.g. NPR/FatFormer (Resize((256, 256))), OmniAID + return v2.Resize((n, n)) + return v2.CenterCrop(n) + elif transform_name == "cap_long_side_1024": # SPAI: native resolution, long side capped at 1024 (PIL) + return CapLongSide(1024) elif transform_name == "to_tensor": return v2.ToTensor() elif transform_name == "to_dtype": - return v2.ToDtype(torch.float32, scale=True) + return ToFloat() elif transform_name == "gaussian_blur": return v2.RandomApply( [v2.GaussianBlur(kernel_size=11, sigma=(1e-3, 3.0))], p=0.5 @@ -144,31 +200,48 @@ class AIGITransformations: elif transform_name == "aide_five_view": # AIDE native-resolution 5-view preprocessing: PIL/tensor -> [5,3,256,256] (DCT patch # selection at native res, then resize the selected patches). Must be the LAST transform. - from models.image.aide.official_arch import AIDEFiveView return AIDEFiveView() elif transform_name == "aide_five_view_train": # Same, but caps the long side at 1024 before the DCT to bound dataloader-worker memory # on large reals (training only; inference uses the uncapped aide_five_view). - from models.image.aide.official_arch import AIDEFiveView return AIDEFiveView(max_side=1024) - elif transform_name == "normalize_dinov2": - return v2.Normalize( - mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225] - ) + # normalize_dinov2: the earlier name of normalize_imagenet (DINOv2 uses ImageNet statistics), kept for old configs. + elif transform_name in ("normalize_imagenet", "normalize_dinov2"): + return v2.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD) elif transform_name == "normalize_clip": - return v2.Normalize( - mean=[0.48145466, 0.4578275, 0.40821073], - std=[0.26862954, 0.26130258, 0.27577711], - ) + return v2.Normalize(mean=CLIP_MEAN, std=CLIP_STD) elif transform_name == "normalize_dinov3": - return v2.Normalize( - mean=(0.430, 0.411, 0.296), - std=(0.213, 0.156, 0.143), - ) + return v2.Normalize(mean=DINOV3_MEAN, std=DINOV3_STD) + # Whole official eval pipelines that are not a chain of the steps above (PIL in, normalized tensor out). + elif transform_name == "effort_cv2_resize_224": # Effort (DeepfakeBench load_rgb): cv2 bicubic squash, PIL out + return Cv2SquareResize(224) + elif transform_name == "cure_eval": # CuRe: PNG->JPEG96, 390 crop, 336 resize, JPEG70, mean=std=0.5 + return CuReTransform() + elif transform_name == "vjepa2_processor": # MoE-JEPA: the V-JEPA 2 video processor on one frame + return VJEPA2ImageTransform() else: raise ValueError(f"Invalid transform name: {transform_name}") - + + def geometric_steps(self) -> list: + """The ``resize_*`` / ``center_crop_*`` steps, in order (they take PIL images too: the web overlay's geometry).""" + return [t for name, t in zip(self.transform_list, self.transforms) if _SIZED.fullmatch(name)] + + def normalize_step(self) -> v2.Normalize | None: + """The ``Normalize`` step, or None for pipelines that feed [0, 1] pixels (or normalize inside a custom step).""" + return next((t for t in self.transforms if isinstance(t, v2.Normalize)), None) + + def without_resize(self) -> "AIGITransformations": + """The same pipeline without its ``resize_*`` / ``center_crop_*`` steps, to score images at native resolution. + Raises ValueError when a remaining step resizes by itself, as the result would then not be native.""" + kept = [name for name in self.transform_list if not _SIZED.fullmatch(name)] + resizing = [name for name in kept if name.startswith(_RESIZING_STEPS)] + if resizing: + raise ValueError(f"{self}: {resizing} resize the image themselves, so it has no native-resolution form") + return AIGITransformations(kept, pil_input=self.pil_input) + + def __repr__(self): + return f"AIGITransformations({self.transform_list})" + def __call__(self, image): # A 4D [T, C, H, W] tensor is a video clip. Wrap it as tv_tensors.Video so the v2 # transforms (crops, flips, normalize, ...) apply with the SAME sampled parameters to @@ -176,6 +249,8 @@ class AIGITransformations: # batch. Single images ([C, H, W] tensors or PIL) pass through unchanged. if isinstance(image, torch.Tensor) and image.ndim == 4: image = tv_tensors.Video(image) + elif self.pil_input and isinstance(image, torch.Tensor) and image.ndim == 3 and image.dtype == torch.uint8: + image = TF.to_pil_image(image) for transform in self.transforms: image = transform(image) return image