Update SynthScope Space from the GitHub repository

LICENSES.md

@@ -40,6 +40,7 @@ third-party components, and every **pretrained checkpoint / backbone** SynthScop
 | `dinov3` | DINOv3 probe | facebookresearch/dinov3 | DINOv3 License (gated) | DINOv3 backbone | **gated** — DINOv3 License |
 | `adapted_dino` | Adapter-tuned DINOv2 (SynthScope) | this repo | Apache-2.0 | — (SynthScope-trained) | n/a |
 | `d3` | D3 (ICCV'25) | ⚠ verify (official repo) | ⚠ verify | training-free (no weights) | n/a |
+| `split` | SPLIT (ECCV'26) | mldljyh/SPLIT | no LICENSE file in the repo: **no code copied**; `models/video/split/split.py` is SynthScope's own implementation of the paper's equations | training-free (no weights) | n/a |
 | `restrav` | ReStraV | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a |
 | `stall` | STALL (CVPR'26) | ⚠ verify (official repo) | ⚠ verify | training-free; needs DINOv3 backbone | gated (DINOv3) |
 
@@ -55,6 +56,7 @@ third-party components, and every **pretrained checkpoint / backbone** SynthScop
 | SigLIP | `co_spy` | Apache-2.0 (verify) |
 | MFM ViT-B/16 | `spai` | ⚠ verify |
 | Stable Diffusion VAE | `co_spy` | CreativeML Open RAIL-M (verify) |
+| X-CLIP B/16 (`microsoft/xclip-base-patch16`) | `d3`, `split` | MIT (HF model card) |
 | LlamaGen VQ-16 tokenizer (`vq_ds16_c2i.pt`, HF `FoundationVision/LlamaGen`) | `d3qe` | MIT |
 | Perception Encoder Core L/14-336 (`facebook/PE-Core-L14-336`) | `cure` | Apache-2.0 (Meta); `official_arch.py` vendors the CuRe/Meta backbone code — see its header |
 

benchmark/bench_eval_video.py

@@ -27,6 +27,7 @@ import json
 import sys
 from pathlib import Path
 
+import numpy as np
 import torch
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "interface"))
@@ -103,20 +104,32 @@ def metrics(fake_probs: torch.Tensor, labels: torch.Tensor):
             "fake_acc": round(fake_acc, 2), "acc": round((real_acc + fake_acc) / 2, 2)}
 
 
+FPRS = (0.001, 0.01, 0.05)
+
+
 @torch.no_grad()
-def score_videos(engine, spec, paths):
+def score_videos(engine, spec, paths, raw=False):
+    """P(fake) per video; with ``raw`` (training-free detectors with a ``score``), the raw score oriented so that higher
+    means generated instead, which keeps the order at the extremes where probabilities saturate."""
     probs = []
     for p in paths:
         try:
-            probs.append(engine.predict_video_clip(spec, str(p))["fake_probability"])
+            probs.append(engine.video_raw_score(spec, str(p)) if raw else engine.predict_video_clip(spec, str(p))["fake_probability"])
         except Exception as e:  # noqa: BLE001
             print(f"    bad video {p.name}: {e}")
     return probs
 
 
+def fake_recall_at_fpr(real_scores, fake_scores):
+    """SPLIT's operating-point metric (official eval.py): the threshold is the (1 - FPR) quantile of the real scores
+    (numpy method "higher"); fake recall = share of fakes scoring at or above it. In percent, per FPR."""
+    real, fake = np.asarray(real_scores, dtype="float64"), np.asarray(fake_scores, dtype="float64")
+    return {f"{a * 100:g}": round(float((fake >= np.quantile(real, 1 - a, method="higher")).mean()) * 100, 2) for a in FPRS}
+
+
 def main(argv=None) -> int:
     ap = argparse.ArgumentParser()
-    ap.add_argument("-m", "--model", required=True, choices=["d3", "restrav", "stall"])
+    ap.add_argument("-m", "--model", required=True, choices=["d3", "split", "restrav", "stall"])
     ap.add_argument("--benchmark", choices=["genvideo", "fakeonly", "pergen"], help="layout (not needed with --dataset)")
     ap.add_argument("--data", help="benchmark root (not needed with --dataset)")
     ap.add_argument("--dataset", help="a supported video dataset (genvideo, videophy, vidprom, evalcrafter, ...): downloads "
@@ -155,6 +168,8 @@ def main(argv=None) -> int:
           f"| device={engine.device} ===", flush=True)
 
     per_gen, agg = {}, {"ap": [], "ap_real": [], "auroc": [], "acc": []}
+    pooled_fake_raw = []                                 # every generator's fakes, for the pooled ("Overall") recall
+    raw = hasattr(engine._get_model(spec), "score")     # training-free detectors: also fake recall at fixed FPR
     # score the shared real pool once if it is identical across generators
     real_cache = {}
     for name, reals, fakes in gens:
@@ -169,11 +184,18 @@ def main(argv=None) -> int:
             rp = score_videos(engine, spec, reals)
             real_cache[key] = rp
         fp = score_videos(engine, spec, fakes)
+        if raw:
+            rraw = real_cache.setdefault(("raw",) + key, None) or score_videos(engine, spec, reals, raw=True)
+            real_cache[("raw",) + key] = rraw
+            fraw = score_videos(engine, spec, fakes, raw=True)
+            pooled_fake_raw += fraw
         probs = torch.tensor(rp + fp)
         labels = torch.tensor([0] * len(rp) + [1] * len(fp))
         m = metrics(probs, labels)
         if m is None:
             print(f"    skip {name}: one class"); continue
+        if raw:
+            m["fake_recall_at_fpr"] = fake_recall_at_fpr(rraw, fraw)
         per_gen[name] = m
         for k in agg:
             agg[k].append(m[k])
@@ -186,6 +208,12 @@ def main(argv=None) -> int:
         return round(sum(xs) / len(xs), 2)
     means = {"acc": mean(agg["acc"]), "map": mean(agg["ap"]), "map_real": mean(agg["ap_real"]),
              "mauroc": mean(agg["auroc"])}
+    if raw:
+        means["fake_recall_at_fpr"] = {k: mean([g["fake_recall_at_fpr"][k] for g in per_gen.values()])
+                                       for k in next(iter(per_gen.values()))["fake_recall_at_fpr"]}
+        # Overall: all fakes against the real pool(s) (each distinct pool once), as the SPLIT paper's "Overall" row
+        pooled_real_raw = [s for k, v in real_cache.items() if k[0] == "raw" for s in v]
+        means["overall_fake_recall_at_fpr"] = fake_recall_at_fpr(pooled_real_raw, pooled_fake_raw)
     result = {"tag": args.tag, "model": args.model, "ckpt": "registry-default",
               "benchmark": f"video:{args.benchmark}", "data_root": str(root),
               "n_generators": len(per_gen), "means": means, "per_generator": per_gen,

conf/model/video/split.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - /model/default
+  - _self_
+
+# SPLIT (mldljyh/SPLIT, ECCV 2026): training-free; frozen X-CLIP patch tokens -> two-step temporal roughness ** gamma
+# x spatial patch-level motion incoherence. Higher score = more likely generated.
+model_type: split
+encoder: xclip-16
+gamma: 8.0
+# Decision threshold on the raw score for argmax accuracy (the engine uses models/video/split/split_calibration.json).
+threshold: 0.0
+num_frames: 16

dataset.py

@@ -25,7 +25,7 @@ from decord import VideoReader, cpu
 from torch.utils.data import DataLoader
 from transformations import AIGITransformations
 from torchvision.io import decode_image, ImageReadMode
-from PIL import Image, ImageFile
+from PIL import Image, ImageFile, ImageSequence
 from config_utils import compose_config
 from models.image.dda.official_train_data import MEAN as DDA_MEAN, STD as DDA_STD, build_item as dda_build_item, \
     create_train_transforms as dda_train_transforms
@@ -71,6 +71,40 @@ def _has_videos(directory: Path) -> bool:
     return directory.is_dir() and any(_iter_videos(directory))
 
 
+class _PILFrames:
+    """The part of decord's ``VideoReader`` API that ``sample_video_frames`` uses, for animated images decord cannot open
+    (GIFs: decord fails with "broken metadata" on them). Every frame is composited to RGB at the canvas size, and the
+    frame rate is the frame count over the summed frame durations (100 ms per frame when a GIF stores none)."""
+
+    class _Array:
+        def __init__(self, a: np.ndarray):
+            self._a = a
+
+        def asnumpy(self) -> np.ndarray:
+            return self._a
+
+    def __init__(self, path):
+        with Image.open(path) as im:
+            frames, ms = [], 0.0
+            for f in ImageSequence.Iterator(im):
+                frames.append(np.asarray(f.convert("RGB")))
+                ms += float(f.info.get("duration") or 100)
+        self._frames = np.stack(frames)
+        self._fps = len(frames) / (ms / 1000.0)
+
+    def __len__(self) -> int:
+        return len(self._frames)
+
+    def get_avg_fps(self) -> float:
+        return self._fps
+
+    def get_batch(self, indices):
+        return self._Array(self._frames[indices])
+
+    def __getitem__(self, i):
+        return self._Array(self._frames[i])
+
+
 def sample_video_frames(
     path: str | Path,
     num_frames: int,
@@ -108,7 +142,11 @@ def sample_video_frames(
         reader = VideoReader(str(path), ctx=cpu(0))
         num_source = len(reader)
     except Exception:
-        return torch.zeros(num_frames, 3, size, size)
+        try:                                               # animated GIFs: decord cannot read them, PIL can
+            reader = _PILFrames(path)
+            num_source = len(reader)
+        except Exception:
+            return torch.zeros(num_frames, 3, size, size)
     if num_source == 0:
         return torch.zeros(num_frames, 3, size, size)
 

docs/CAPABILITIES.md

@@ -36,6 +36,7 @@ matching `conf/model/<modality>/<key>.yaml`, a `trainer.py._build_model` branch,
 | `effort` (+ `effort_chameleon`) | image | ce | official | – | CLIP ViT-L/14 + SVD-residual attention (orthogonal subspace decomposition) |
 | `vib_net` | image | vib | synthscope | – | Variational information bottleneck on frozen CLIP ViT-L/14 (trained in SynthScope; reproduces the paper) |
 | `d3` | video | video | training-free | – | Disentangled detail dynamics (X-CLIP) |
+| `split` | video | video | training-free | – | Temporal roughness x spatial patch-level motion incoherence (X-CLIP) |
 | `restrav` | video | video | synthscope | – | Representation-trajectory geometry (DINOv2) |
 | `stall` | video | video | training-free | – | Spatio-temporal whitened log-likelihoods (DINOv3) |
 

interface/backend/build_repro_full.py

@@ -118,11 +118,14 @@ VAL = {
     # robust metrics (the mAP harness is degenerate: 10000 reals : as few as 56 fakes/generator).
     # D3 (training-free): "synthscope" = SynthScope's D3, "official" = the official D3 code end to end on the same data
     # (repro/official_evals/d3_official_eval.py). mAP = AP with real as the positive class, as the paper reports.
-    ("d3", "GenVideo"):                  {"synthscope": {"map": 99.14, "auroc": 98.46, "acc": 96.15}, "official": {"map": 97.93, "auroc": 97.04}},
+    ("d3", "GenVideo"):                  {"synthscope": {"map": 98.95, "auroc": 98.23, "acc": 95.68}, "official": {"map": 97.93, "auroc": 97.04}},
     ("d3", "EvalCrafter"):               {"synthscope": {"map": 99.3, "auroc": 98.62, "acc": 96.59}, "official": {"map": 98.7, "auroc": 97.81}},
     ("d3", "VideoPhy"):                  {"synthscope": {"map": 99.45, "auroc": 98.09, "acc": 95.19}, "official": {"map": 98.84, "auroc": 96.28}},
     ("d3", "VidProM"):                   {"synthscope": {"map": 92.15, "auroc": 90.86, "acc": 84.86}, "official": {"map": 84.91, "auroc": 82.83}},
-    ("restrav", "GenVidBench"):          {"synthscope": {"map": 92.7, "auroc": 95.9, "acc": 86.87}},
+    # SPLIT (training-free): the paper's metric is fake recall at fixed FPR (in the spec note); these are AP (fake positive) /
+    # AUROC / acc at the calibrated threshold. synthscope = job 270384, official = official code end to end (job 270323).
+    ("split", "GenVideo"):               {"synthscope": {"map": 99.79, "auroc": 99.97, "acc": 99.41}, "official": {"map": 96.12, "auroc": 99.16}},
+    ("restrav", "GenVidBench"):         {"synthscope": {"map": 92.7, "auroc": 95.9, "acc": 86.87}},
     ("stall", "GenVideo"):               {"synthscope": {"map": 82.38, "auroc": 83.51, "acc": 56.88}},
     ("stall", "VideoFeedback"):          {"synthscope": {"map": 83.07, "auroc": 85.1, "acc": 62.12}},
     ("stall", "ComGenVid"):              {"synthscope": {"map": 82.59, "auroc": 84.0, "acc": 57.34}},

interface/backend/engine.py

@@ -125,6 +125,9 @@ def _build_model(cfg):
     if model_type == "d3":
         from models.video.d3.d3 import SynthScopeD3Model
         return SynthScopeD3Model(cfg)
+    if model_type == "split":
+        from models.video.split.split import SynthScopeSPLITModel
+        return SynthScopeSPLITModel(cfg)
     if model_type == "restrav":
         from models.video.restrav.restrav import SynthScopeReStraVModel
         return SynthScopeReStraVModel(cfg)
@@ -261,25 +264,29 @@ class Engine:
             return torch.sigmoid(logit.reshape(x.shape[0]))
         raise ValueError(f"Unexpected image predict_kind: {kind}")
 
-    def _d3_calibration(self):
-        """Load (and cache) the D3 score->probability calibration if present."""
-        if not hasattr(self, "_d3_cal"):
+    def _video_calibration(self, key: str):
+        """Load (and cache) a training-free video detector's raw score -> probability calibration, if it has one: a
+        ``<key>_calibration.json`` in the checkpoint folder overrides the one shipped in ``models/video/<key>/``."""
+        if not hasattr(self, "_video_cal"):
+            self._video_cal = {}
+        if key not in self._video_cal:
             import json
-            # A file in the checkpoint folder overrides the one shipped with the model (fitted on GenVideo-Val).
-            p = config.CHECKPOINTS_DIR / "d3_calibration.json"
+            p = config.CHECKPOINTS_DIR / f"{key}_calibration.json"
             if not p.exists():
-                p = config.REPO_ROOT / "models" / "video" / "d3" / "d3_calibration.json"
-            self._d3_cal = json.loads(p.read_text()) if p.exists() else None
-        return self._d3_cal
+                p = config.REPO_ROOT / "models" / "video" / key / f"{key}_calibration.json"
+            self._video_cal[key] = json.loads(p.read_text()) if p.exists() else None
+        return self._video_cal[key]
 
     @torch.no_grad()
     def _video_fake_prob(self, model, spec: ModelSpec, clip: torch.Tensor) -> torch.Tensor:
         """Return P(fake) for a single normalized clip tensor [T,3,H,W]."""
         frames = clip.unsqueeze(0).to(self.device)  # [1,T,3,H,W]
-        if spec.key == "d3":
-            cal = self._d3_calibration()
+        if spec.key in ("d3", "split"):
+            cal = self._video_calibration(spec.key)
             if cal is not None:
-                s = model.score(frames)  # [1] raw volatility score
+                s = model.score(frames)  # [1] raw score (D3 volatility, SPLIT roughness x incoherence)
+                if cal.get("log_score"):  # SPLIT's score spans many orders of magnitude: calibrated on its log
+                    s = s.clamp_min(1e-30).log()
                 z = (s - cal["threshold"]) / cal["scale"]
                 if not cal["higher_is_fake"]:
                     z = -z
@@ -408,6 +415,14 @@ class Engine:
         res["num_frames"] = int(clip.shape[0])
         return res
 
+    @torch.no_grad()
+    def video_raw_score(self, spec: ModelSpec, path) -> float:
+        """A training-free video detector's raw score for one clip, oriented so that higher means more likely generated
+        (for fake recall at a fixed false-positive rate, where saturated probabilities would tie)."""
+        model = self._get_model(spec)
+        s = float(model.score(self.video_clip(spec, path).unsqueeze(0).to(self.device)).item())
+        return s if getattr(model, "higher_is_fake", True) else -s
+
     def _result(self, spec: ModelSpec, fake_prob: float, elapsed: float) -> dict:
         fake_prob = max(0.0, min(1.0, fake_prob))
         return {

interface/backend/registry.py

@@ -461,6 +461,17 @@ _add(ModelSpec(
           "Official preprocessing (D3 utils/video2frame.py + data/datasets.py): 16 frames at 8 fps (8 for clips under "
           "2 s), BGR channels as cv2 reads them, 10% crop of the longer side, resize 224, ImageNet normalization.",
 ))
+_add(ModelSpec(
+    key="split", name="SPLIT (video)", paper="ECCV'26",
+    modality="video", predict_kind="video", checkpoint=None, ckpt_kind="none", weights="training_free",
+    overrides=["model@ModelConfig=video/split"],
+    num_frames=16, fps=8.0, frame_size=224, video_norm=(IMAGENET_MEAN, IMAGENET_STD), edge_crop=0.1,
+    edge_crop_long_side=True, short_clip_frames=8, fps_nearest=True, bgr=True,
+    notes="Training-free: frozen X-CLIP-B/16 patch tokens; two-step temporal roughness (gamma 8) times spatial "
+          "patch-level motion incoherence is the score. Official preprocessing (the same video2frame / read_video as "
+          "D3): 16 frames at 8 fps (8 for clips under 2 s), BGR, 10% crop of the longer side, resize 224, ImageNet "
+          "normalization.",
+))
 _add(ModelSpec(
     key="restrav", name="ReStraV (video)", paper="2025",
     modality="video", predict_kind="video", checkpoint="restrav.pth", ckpt_kind="synthscope",

interface/backend/repro_full.json

@@ -346,11 +346,11 @@
       "benchmark": "GenVideo (DeMamba GenVideo-Val, Table 1)",
       "metric": "AP",
       "paper": "mAP=98.46",
-      "ours": "SynthScope mAP=99.14 / acc=96.15 / AUROC=98.46 \u00b7 official mAP=97.93 / AUROC=97.04",
-      "ours_synthscope": "mAP=99.14 / acc=96.15 / AUROC=98.46",
+      "ours": "SynthScope mAP=98.95 / acc=95.68 / AUROC=98.23 \u00b7 official mAP=97.93 / AUROC=97.04",
+      "ours_synthscope": "mAP=98.95 / acc=95.68 / AUROC=98.23",
       "ours_official": "mAP=97.93 / AUROC=97.04",
       "status": "reproduced",
-      "note": "D3 protocol (--limit 1000: first 1000 clips per class per generator; AP with real as the positive class). SynthScope engine 99.14 AP (AUROC 98.46, balanced acc 96.15 at the refitted calibration; tag d3_genvideo, job 252246). Official D3 code end to end: 97.93 (tag d3_official_genvideo, job 252271): 9 of 10 generators within 0.25 of the paper; WildScrape 92.50 vs 96.49 (282 GIFs whose length decord"
+      "note": "D3 protocol (--limit 1000: first 1000 clips per class per generator; AP with real as the positive class). SynthScope engine 98.95 AP (AUROC 98.23, balanced acc 95.68 at the refitted calibration; tag d3_genvideo, job 270428, after the GIF decode fix: WildScrape 96.33 vs paper 96.49, was 98.21 when GIFs decoded as black clips). Official D3 code end to end: 97.93 (tag d3_official_genvideo, job 252271"
     },
     {
       "benchmark": "EvalCrafter (Table 2, 14 generators, fake-only)",
@@ -1113,6 +1113,38 @@
       "note": "SERVED native-res path (official weights in SynthScope class). bench_eval --model spai (tag spai_served_synthbuster, 9 gens, limit 200/gen) = 96.75 mAP / 96.97 mAUROC (was 55.67 via the old fixed-448 path)."
     }
   ],
+  "split": [
+    {
+      "benchmark": "GenVideo (GenVideo-Val, MSR-VTT real calibration, supp. Table 18)",
+      "metric": "fake recall @ FPR 0.1/1/5% (Overall = all fakes pooled)",
+      "paper": "fake_recall_fpr0.1=82.13 / fake_recall_fpr1=91.69 / fake_recall_fpr5=97.2",
+      "ours": "SynthScope mAP=99.79 / acc=99.41 / AUROC=99.97 \u00b7 official mAP=96.12 / AUROC=99.16",
+      "ours_synthscope": "mAP=99.79 / acc=99.41 / AUROC=99.97",
+      "ours_official": "mAP=96.12 / AUROC=99.16",
+      "status": "reproduced",
+      "note": "Official code on our data (JPEG frame round-trip, job 270323) reproduces the paper: Overall fake recall 82.22/90.17/96.89 vs 82.13/91.69/97.20. SynthScope decodes frames directly (no JPEG) and reaches 98.87/99.86/99.98 (job 270384; mAP 99.79, AUROC 99.97, acc 99.41 at the calibrated threshold)."
+    },
+    {
+      "benchmark": "FakeParts (partial edits; Tables 1, 15-16)",
+      "metric": "fake recall @ FPR",
+      "paper": "fake_recall_fpr0.1=74.45",
+      "ours": "",
+      "ours_synthscope": "",
+      "ours_official": "",
+      "status": "missing",
+      "note": "not in SynthScope yet"
+    },
+    {
+      "benchmark": "ViF-Bench (Tables 3, 19)",
+      "metric": "fake recall @ FPR",
+      "paper": "fake_recall_fpr0.1=84.52",
+      "ours": "",
+      "ours_synthscope": "",
+      "ours_official": "",
+      "status": "missing",
+      "note": "not in SynthScope yet"
+    }
+  ],
   "stall": [
     {
       "benchmark": "GenVideo (DeMamba GenVideo-Val, 10 generators + MSR-VTT)",

interface/scripts/calibrate_split.py

@@ -0,0 +1,65 @@
+"""Calibrate the training-free SPLIT video detector so its output is a usable probability.
+
+SPLIT's raw score (TTR ** 8 * motion incoherence) spans many orders of magnitude, so the calibration works on its
+logarithm and follows the paper's real-only protocol: the decision threshold is the score at a target false-positive
+rate on real videos alone (default 1%, numpy's ``quantile(..., method="higher")`` as the official ``eval.py``), and the
+logistic scale puts the median real video at P(fake) = 0.05. The engine then reports
+``sigmoid((log(score) - threshold) / scale)`` as P(fake). Fakes are scored only to report the recall at that threshold.
+
+  conda run -n SynthScope python interface/scripts/calibrate_split.py [--n-real 2000] [--n-fake 1000] [--fpr 0.01]
+      [--data DIR] [--out FILE]
+
+Clips go through the engine's own SPLIT preprocessing (``Engine.video_clip``), so the calibration matches what the web
+app and benchmarks feed the model. ``--data`` takes GenVideo-Val as released (``Real/`` + ``Fake/<gen>/``) or as laid out
+by the ``genvideo`` loader (``test/<gen>/{0_real,1_fake}/``).
+"""
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+import numpy as np
+
+sys.path[:0] = [str(Path(__file__).resolve().parents[1]), str(Path(__file__).resolve().parent)]
+from backend import config                     # noqa: E402
+from backend.engine import get_engine           # noqa: E402
+from backend.registry import get_spec            # noqa: E402
+from calibrate_d3 import GENVIDEO, collect, score_paths   # noqa: E402
+
+EPS = 1e-30
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--n-real", type=int, default=2000)
+    ap.add_argument("--n-fake", type=int, default=1000)
+    ap.add_argument("--fpr", type=float, default=0.01, help="target false-positive rate on real videos")
+    ap.add_argument("--data", default=str(GENVIDEO), help="GenVideo-Val root (released layout or the genvideo loader's)")
+    ap.add_argument("--out", default=None, help="where to write the JSON (default: <CKPT_DIR>/split_calibration.json)")
+    args = ap.parse_args()
+
+    engine = get_engine()
+    spec = get_spec("split")
+    model = engine._get_model(spec)
+    reals, _ = collect(args.n_real, args.data)
+    _, fakes = collect(args.n_fake, args.data)
+    print(f"scoring {len(reals)} real + {len(fakes)} fake clips on {engine.device}...")
+    rs = np.log(np.maximum(score_paths(engine, spec, model, reals), EPS))
+    fs = np.log(np.maximum(score_paths(engine, spec, model, fakes), EPS))
+
+    threshold = float(np.quantile(rs, 1 - args.fpr, method="higher"))
+    scale = max((threshold - float(np.median(rs))) / math.log(19), 1e-6)    # median real -> sigmoid(-ln 19) = 0.05
+    cal = {"threshold": threshold, "scale": scale, "higher_is_fake": True, "log_score": True,
+           "target_fpr": args.fpr, "real_fpr": float((rs >= threshold).mean()),
+           "fake_recall": float((fs >= threshold).mean()), "median_log_real": float(np.median(rs)),
+           "median_log_fake": float(np.median(fs)), "n_real": int(len(rs)), "n_fake": int(len(fs))}
+    out = Path(args.out) if args.out else config.CHECKPOINTS_DIR / "split_calibration.json"
+    out.write_text(json.dumps(cal, indent=2) + "\n")
+    print(f"threshold(log)={threshold:.4f} scale={scale:.4f} real FPR={cal['real_fpr']:.4f} "
+          f"fake recall={cal['fake_recall']:.4f}")
+    print(f"wrote {out}")
+
+
+if __name__ == "__main__":
+    main()

models/video/d3/README.md

@@ -149,7 +149,7 @@ frame extraction, reader and model run end to end on the same data (`repro/offic
 
 | Benchmark (AP, real positive) | Paper | Official code, our data | SynthScope | Status |
 |---|---:|---:|---:|---|
-| GenVideo-Val (Table 1, 10 generators) | 98.46 | 97.93 | **99.14** | reproduced |
+| GenVideo-Val (Table 1, 10 generators) | 98.46 | 97.93 | **98.95** | reproduced |
 | EvalCrafter (Table 2, 14 generators) | 98.87 | 98.70 | **99.30** | reproduced |
 | VideoPhy (Table 3; 8 of 10 generators still downloadable) | 99.16 (8 shown: 99.22) | 98.84 | **99.45** | reproduced |
 | VidProM (Table 4; 4 of 6 generators, first 1000 clips each) | 88.46 (those 4: 84.95) | 84.91 | 92.15 | reference (subsample) |
@@ -158,11 +158,14 @@ frame extraction, reader and model run end to end on the same data (`repro/offic
   are within 0.25 AP; VidProM's four generators average 84.91 vs 84.95 (Text2Video-Zero 46.36 vs 45.11, the paper's
   noted failure case). The one outlier is GenVideo's WildScrape (92.50 vs 96.49): its 282 GIFs cannot be timed by
   decord, so their frames start at 0 instead of the official random offset.
+- **GIFs (2026-11-01):** decord cannot open GIFs at all, and SynthScope's decoder used to return a black clip for them, which
+  D3 (flat = generated) called fake for free. Decoded with PIL now, WildScrape is 96.33 AP (paper 96.49; it was 98.21 on
+  black clips) and GenVideo 98.95 (was 99.14); the other nine generators are unchanged (job 270428).
 - **SynthScope's D3 is the official model**: on the frames the official code extracts it returns identical scores
   (max |diff| 0). Its numbers differ from the official column only because the official pipeline writes frames to JPEG
   with ffmpeg and reads them back; SynthScope decodes frames directly. That matters most for low-quality clips
   (VidProM Text2Video-Zero: 71.28 in SynthScope vs 46.36 with the official JPEG round-trip).
-- AUROC and balanced accuracy (at the calibrated threshold) for SynthScope: GenVideo 98.46 / 96.15, EvalCrafter
+- AUROC and balanced accuracy (at the calibrated threshold) for SynthScope: GenVideo 98.23 / 95.68, EvalCrafter
   98.62 / 96.59, VideoPhy 98.09 / 95.19, VidProM 90.86 / 84.86. Per-generator numbers are in
   `repro/benchmark_specs/d3.json` and `repro/bench_results/d3_*.json`.
 

models/video/d3/d3.py

@@ -100,6 +100,8 @@ class _ResnetEncoder(nn.Module):
 class SynthScopeD3Model(nn.Module):
     """Training-free second-order video detector (D3)."""
 
+    higher_is_fake = False    # real videos have the larger second-order volatility
+
     def __init__(self, config: DictConfig):
         super().__init__()
         self.config = config

models/video/split/README.md

@@ -0,0 +1,95 @@
+# SPLIT (`split`)
+
+**SPLIT: Training-Free AI-Generated and Partially Edited Video Detection via Spatial Patch-Level Incoherence and
+Temporal Roughness** (Jongyeop Hyun, Hyounghun Kim; ECCV 2026) — [paper (arXiv 2607.02886)](https://arxiv.org/abs/2607.02886) ·
+[official code](https://github.com/mldljyh/SPLIT)
+
+A training-free video detector. Real motion is smooth in a frozen encoder's patch-feature space and moves coherently
+across neighbouring patches; generated (or partially edited) video jitters from frame to frame and moves incoherently.
+SPLIT measures both on patch tokens and multiplies them. The paper evaluates it where deployment needs it: fake recall
+at a fixed, very low false-positive rate on real videos (0.1 / 1 / 5%), with the threshold calibrated on real videos only.
+
+## Pipeline
+
+```
+video ─ 16 frames at 8 fps (8 for clips under 2 s), BGR, 10% crop of the longer side, resize 224, ImageNet norm
+      → frozen X-CLIP B/16 vision tower → patch tokens P[t, n] (CLS dropped)
+      → TTR  = mean over patches of log2(L1 / L2)          L1 = Σ‖P[t+1] − P[t]‖ (1-frame steps)
+                                                           L2 = Σ‖P[t+2] − P[t]‖ / 2 · (T−1)/(T−2) (2-frame steps)
+      → LSMI = mean ‖spatial difference of the motion field P[t+1] − P[t] on the patch grid‖ (x and y averaged)
+      → score = TTR ** 8 · LSMI          (higher = more likely generated)
+```
+
+## SynthScope adaptations
+
+| choice | reason |
+|---|---|
+| `SynthScopeSPLITModel` is SynthScope's own implementation of the paper's equations | The official repository has no license, so no code is copied. It returns the official `SPLIT_model.forward_score` on the same frames (relative difference ≤ 7e-6, job 269966). |
+| Registry entry reuses D3's video preprocessing | SPLIT's released code reads frames exactly as D3 does (the same `video2frame` / `read_video`): 16 frames at 8 fps, BGR, 10% crop of the long side, resize 224, ImageNet normalization. |
+| Frames are decoded directly (decord; PIL for GIFs), not written to JPEG by ffmpeg and read back | The official pipeline's lossy JPEG round-trip is the one remaining difference from the official code, and it changes the numbers a lot (see Results). |
+| Clips longer than 3 s start at 0 | The official extraction picks a random start in a multiprocessing pool, so its frames are not reproducible anyway. |
+| `split_calibration.json` (P(fake) for the web app) | The raw score spans many orders of magnitude, so it is calibrated on its log, by the paper's real-only rule (see below). |
+| One dummy `_noop` parameter | Lets the SynthScope trainer run on a training-free model; it has no effect. |
+
+## Results
+
+Paper metric: **fake recall (%) at FPR 0.1 / 1 / 5%**, the threshold being the (1 − FPR) quantile of the real scores
+(official `eval.py`). GenVideo-Val: its 10,000 MSR-VTT real videos calibrate the threshold, and all 8,584 fakes of the
+10 generators are scored (the official reader drops 11 clips with fewer than 8 frames) (paper Table 18, "thresholds calibrated using the MSR-VTT real set only").
+
+Two columns besides the paper: **official code, our data** runs the official frame extraction (ffmpeg JPEG frames),
+reader, `SPLIT_model` and `evaluate_split` end to end on our copy of GenVideo-Val
+(`repro/official_evals/split_official_eval.py`, job 270323); **SynthScope** is this implementation
+(`benchmark/bench_eval_video.py -m split --dataset genvideo`, job 270384).
+
+| GenVideo (fake recall @ FPR 0.1 / 1 / 5%) | Paper | Official code, our data | SynthScope |
+|---|---|---|---|
+| **Overall** | 82.13 / 91.69 / 97.20 | **82.22 / 90.17 / 96.89** | 98.87 / 99.86 / 99.98 |
+| Crafter | 88.84 / 96.78 / 99.64 | 89.99 / 96.21 / 99.64 | 100 / 100 / 100 |
+| Gen2 | 53.26 / 77.61 / 94.71 | 56.16 / 76.30 / 94.64 | 100 / 100 / 100 |
+| HotShot | 99.86 / 100 / 100 | 99.86 / 100 / 100 | 100 / 100 / 100 |
+| Lavie | 92.50 / 96.36 / 98.79 | 93.21 / 96.07 / 98.71 | 99.93 / 99.93 / 100 |
+| ModelScope | 86.71 / 93.57 / 97.86 | 87.29 / 93.14 / 97.86 | 99.57 / 99.86 / 100 |
+| MoonValley | 49.36 / 76.36 / 95.37 | 52.88 / 74.76 / 95.37 | 100 / 100 / 100 |
+| MorphStudio | 94.71 / 98.43 / 99.71 | 95.00 / 98.00 / 99.71 | 100 / 100 / 100 |
+| Show_1 | 96.86 / 99.86 / 100 | 97.86 / 99.71 / 100 | 100 / 100 / 100 |
+| Sora | 87.50 / 98.21 / 100 | 91.07 / 98.21 / 100 | 100 / 100 / 100 |
+| WildScrape | 71.66 / 79.76 / 85.89 | 73.27 / 79.74 / 86.42 | 89.94 / 98.92 / 99.78 |
+| D3 Overall (the paper's baseline row) | 3.00 / 55.75 / 87.42 | — | 0.24 / 63.32 / 96.41 |
+
+- **The official code reproduces the paper on our data**: Overall within 1.6 points at every FPR, every generator within
+  0.6 at FPR 5%, and the larger 0.1% differences (Gen2, MoonValley, Sora with 56 fakes) are the random clip start the
+  official extraction draws.
+- **SynthScope scores higher because it skips the JPEG round-trip.** The official `video2frame.py` writes every frame
+  with ffmpeg's default JPEG settings and reads it back; the compression noise differs from frame to frame, which adds
+  temporal roughness to real videos and stretches the real-score tail that sets the threshold. On the same videos,
+  decoded directly, SPLIT separates real and generated almost perfectly (AP 99.79, AUROC 99.97 averaged over generators).
+  The **official code, our data** column is the like-for-like comparison with the paper.
+- AP / AUROC / balanced accuracy at the calibrated threshold (SynthScope, mean over generators): 99.79 / 99.97 / 99.41.
+  Per-generator numbers: `repro/bench_results/split_genvideo_fpr.json`, `split_official_genvideo.json`.
+- Not run: FakeParts and ViF-Bench (the paper's other two benchmarks; not in SynthScope yet), and the cross-real
+  threshold transfer to ROVI (needs FakeParts' real set).
+
+## Probability calibration (`split_calibration.json`)
+
+The raw score (TTR⁸ · LSMI) has a median of about 1e-10 on real GenVideo videos and about 1 on generated ones, so the engine maps
+its logarithm to P(fake) = sigmoid((log score − threshold) / scale). Following the paper's real-only protocol, the
+threshold is the log score at FPR 1% on 2,000 GenVideo-Val real videos (P(fake) = 0.5 there), and the scale puts the
+median real video at P(fake) = 0.05 (`interface/scripts/calibrate_split.py`, through the engine's own preprocessing; on
+1,000 random GenVideo fakes the recall at that threshold is 100%). A `split_calibration.json` in `$SYNTHSCOPE_CKPT_DIR`
+overrides the shipped one. AP, AUROC and fake recall at fixed FPR do not depend on it.
+
+## Usage
+
+```bash
+synthscope detect path/to/video.mp4 -m split
+python benchmark/bench_eval_video.py -m split --dataset genvideo --tag split_genvideo_fpr
+```
+
+`bench_eval_video.py` reports fake recall at FPR 0.1 / 1 / 5% (per generator, their mean, and pooled over all
+generators) for training-free detectors that expose a raw `score`.
+
+## License
+
+SynthScope's implementation is Apache-2.0. The official SPLIT repository has no license; no code from it is included.
+The X-CLIP weights (`microsoft/xclip-base-patch16`) are MIT.

models/video/split/split.py

@@ -0,0 +1,92 @@
+"""SPLIT integrated into SynthScope.
+
+Paper: "SPLIT: Training-Free AI-Generated and Partially Edited Video Detection via Spatial Patch-Level Incoherence and
+Temporal Roughness" (Hyun & Kim, ECCV 2026). https://github.com/mldljyh/SPLIT
+
+Training-free. A frozen vision encoder (X-CLIP B/16 by default) turns every frame into patch tokens P[t, n] (CLS
+dropped). Two signals are computed on them:
+
+* Two-step Temporal Roughness (TTR), per patch: the path length of its token trajectory with 1-frame steps,
+  L1 = sum_t ||P[t+1] - P[t]||, against the path length with 2-frame steps rescaled to the same number of steps,
+  L2 = (sum_t ||P[t+2] - P[t]|| / 2) * (T-1) / (T-2); TTR = log2(L1 / L2). Real motion is smooth (TTR near 0); generated
+  video jitters from frame to frame, so 1-frame steps are long compared with 2-frame steps. Averaged over patches.
+* Spatial patch-level motion incoherence: the motion field M[t] = P[t+1] - P[t] on the patch grid and its spatial
+  differences between neighbouring patches; the mean of their L2 norms over time and space, averaged over x and y.
+
+score = TTR ** gamma * incoherence (gamma = 8); higher means more likely generated.
+
+This is SynthScope's own implementation of the paper's equations (the official repository has no license); it is
+checked against the official ``SPLIT_model`` on the same frames (repro/split parity script, identical scores).
+The released code reads frames exactly as D3 does (the same ``video2frame`` / ``read_video``), so the registry entry
+reuses D3's preprocessing: 16 frames at 8 fps, BGR, 10% crop of the long side, resize 224, ImageNet normalization.
+"""
+
+from __future__ import annotations
+
+import math
+
+import torch
+import torch.nn as nn
+from omegaconf import DictConfig
+from transformers import XCLIPVisionModel
+
+ENCODERS = {"xclip-16": "microsoft/xclip-base-patch16", "xclip-32": "microsoft/xclip-base-patch32"}
+
+
+class SynthScopeSPLITModel(nn.Module):
+    """Training-free SPLIT detector on a frozen X-CLIP vision tower."""
+
+    higher_is_fake = True     # orientation of ``score`` (benchmark/bench_eval_video.py fake recall at fixed FPR)
+
+    def __init__(self, config: DictConfig):
+        super().__init__()
+        cfg = config.ModelConfig
+        self.encoder = XCLIPVisionModel.from_pretrained(ENCODERS[str(cfg.get("encoder", "xclip-16")).lower()])
+        self.encoder.requires_grad_(False)
+        self.encoder.eval()
+        self.gamma = float(cfg.get("gamma", 8.0))
+        self.threshold = float(cfg.get("threshold", 0.0))
+        self._noop = nn.Parameter(torch.zeros(1))      # the trainer's AdamW needs one parameter; training is a no-op
+
+    def train(self, mode: bool = True):
+        super().train(mode)
+        self.encoder.eval()                            # the encoder stays frozen in eval mode
+        return self
+
+    def patch_tokens(self, frames: torch.Tensor) -> torch.Tensor:
+        """``frames`` [B, T, 3, H, W] (normalized) -> patch tokens [B, T, N, D] (CLS dropped)."""
+        b, t = frames.shape[:2]
+        with torch.no_grad():
+            tokens = self.encoder(pixel_values=frames.reshape(b * t, *frames.shape[2:])).last_hidden_state[:, 1:]
+        return tokens.reshape(b, t, *tokens.shape[1:]).float()
+
+    @staticmethod
+    def temporal_roughness(p: torch.Tensor) -> torch.Tensor:
+        """Mean over patches of TTR = log2(L1 / L2); ``p`` [B, T, N, D] -> [B]."""
+        t = p.shape[1]
+        l1 = (p[:, 1:] - p[:, :-1]).norm(dim=-1).sum(dim=1)                        # [B, N]
+        l2 = (p[:, 2:] - p[:, :-2]).norm(dim=-1).sum(dim=1) / 2 * (t - 1) / (t - 2)
+        return ((torch.log(l1 + 1e-8) - torch.log(l2 + 1e-8)) / math.log(2)).mean(dim=1)
+
+    @staticmethod
+    def motion_incoherence(p: torch.Tensor) -> torch.Tensor:
+        """Mean L2 norm of the spatial differences of the motion field on the patch grid; ``p`` [B, T, N, D] -> [B]."""
+        b, t, n, d = p.shape
+        g = int(round(n ** 0.5))
+        m = (p[:, 1:] - p[:, :-1]).reshape(b, t - 1, g, g, d)                        # motion field on the grid
+        dx = (m[:, :, :, :-1] - m[:, :, :, 1:]).norm(dim=-1).mean(dim=(1, 2, 3))
+        dy = (m[:, :, :-1] - m[:, :, 1:]).norm(dim=-1).mean(dim=(1, 2, 3))
+        return (dx + dy) / 2
+
+    def score(self, frames: torch.Tensor) -> torch.Tensor:
+        """Per-video SPLIT score [B] (higher = more likely generated)."""
+        p = self.patch_tokens(frames)
+        return self.temporal_roughness(p) ** self.gamma * self.motion_incoherence(p)
+
+    def forward(self, frames: torch.Tensor) -> torch.Tensor:
+        s = self.score(frames)
+        return torch.stack([self.threshold - s, s - self.threshold], dim=1)
+
+    def compute_loss(self, batch):
+        frames, _labels, _folder_ids = batch
+        return {"loss": (self._noop * 0.0).sum(), "outputs": self.forward(frames)}

models/video/split/split_calibration.json

@@ -0,0 +1,13 @@
+{
+  "threshold": -12.128589335463142,
+  "scale": 3.5924194383097205,
+  "higher_is_fake": true,
+  "log_score": true,
+  "target_fpr": 0.01,
+  "real_fpr": 0.01,
+  "fake_recall": 1.0,
+  "median_log_real": -22.706249159137492,
+  "median_log_fake": 0.12425237563573246,
+  "n_real": 2000,
+  "n_fake": 1000
+}

repro/MASTER_RESULTS.jsonl

@@ -370,3 +370,9 @@
 {"date": "2026-10-30T22:33:19", "tag": "effort_evalgen", "model": "effort", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/EvalGEN/test", "n_generators": 5, "means": {"acc": 92.16, "map": 99.55, "mauroc": 99.16}}
 {"date": "2026-10-30T22:48:42", "tag": "effort_realchain", "model": "effort", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RealChain/test", "n_generators": 7, "means": {"acc": 50.56, "map": 60.38, "mauroc": 64.22}}
 {"date": "2026-10-30T23:19:04", "tag": "effort_rrdataset", "model": "effort", "ckpt": "registry-default", "benchmark": "pergen", "data_root": "/ssdstore/ssalehi/SynthScope/RRDataset/test", "n_generators": 3, "means": {"acc": 64.59, "map": 69.78, "mauroc": 68.6}}
+{"date": "2026-10-31T17:58:11", "tag": "split_genvideo_fpr", "model": "split", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 50.0, "map": 97.05, "map_real": 99.72, "mauroc": 98.42, "fake_recall_at_fpr": {"0.1": 96.66, "1": 96.91, "5": 96.93}}}
+{"date": "2026-10-31T19:33:09", "tag": "split_official_genvideo", "model": "split", "ckpt": "official SPLIT code (XCLIP-16), training-free", "benchmark": "video:official-split", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"overall_fake_recall_at_fpr": {"0.1": 82.22, "1": 90.17, "5": 96.89}, "mean_fake_recall_at_fpr": {"0.1": 83.66, "1": 91.21, "5": 97.23}, "map": 96.12}}
+{"date": "2026-10-31T20:38:46", "tag": "split_genvideo_fpr", "model": "split", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 50.0, "map": 97.05, "map_real": 99.72, "mauroc": 98.42, "fake_recall_at_fpr": {"0.1": 96.66, "1": 96.91, "5": 96.93}, "overall_fake_recall_at_fpr": {"0.1": 96.41, "1": 96.67, "5": 96.69}}}
+{"date": "2026-10-31T22:35:53", "tag": "split_genvideo_fpr", "model": "split", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 99.41, "map": 99.79, "map_real": 100.0, "mauroc": 99.97, "fake_recall_at_fpr": {"0.1": 98.94, "1": 99.87, "5": 99.98}, "overall_fake_recall_at_fpr": {"0.1": 98.87, "1": 99.86, "5": 99.98}}}
+{"date": "2026-10-31T23:42:33", "tag": "d3_genvideo_fpr", "model": "d3", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 95.53, "map": 68.84, "map_real": 99.87, "mauroc": 98.45, "fake_recall_at_fpr": {"0.1": 0.21, "1": 62.03, "5": 96.52}, "overall_fake_recall_at_fpr": {"0.1": 0.24, "1": 63.32, "5": 96.41}}}
+{"date": "2026-11-01T00:56:39", "tag": "d3_genvideo", "model": "d3", "ckpt": "registry-default", "benchmark": "video:pergen", "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test", "n_generators": 10, "means": {"acc": 95.68, "map": 91.6, "map_real": 98.95, "mauroc": 98.23, "fake_recall_at_fpr": {"0.1": 0.0, "1": 33.77, "5": 96.71}, "overall_fake_recall_at_fpr": {"0.1": 0.0, "1": 35.2, "5": 96.34}}}

repro/RESULTS_SUMMARY.md

@@ -23,7 +23,7 @@
 | DDA | AIGCDetect / ForenSynths / EvalGEN / SynthWildX / WildRF / GenImage | 87.8 / 81.4 / 97.2 / 90.9 / 90.3 / 91.7 acc | 87.84 / 81.43 / 96.86 / 90.80 / 90.20 / 89.37 acc | 89.58 / 80.79 / 96.79 / 88.00 / 90.56 / **91.35** acc | ✅ official match; trained in SynthScope reproduces 6 of 7 incl. GenImage (SynthWildX 2.9 below) |
 | D3QE | ARForensics / ForenSynths-7 / GenImage-6 | 82.11 / 83.73 / 78.61 acc | 81.74 / 83.45 / 79.44 acc | 79.83 / 82.14 / 77.01 acc | ✅ official match; trained in SynthScope: AP matches, acc 1.6-2.3 below |
 | OmniAID | GenImage / AIGCDetect / Mirage-Test (GenImage release) | 95.94 / 88.87 / 51.10 acc | 95.82 / 88.87 / 51.10 acc | — | ✅ official match (v1 release too: 97.16 / 92.88 / 88.39 vs 97.24 / 92.88 / 88.39) |
-| D3 (video) | GenVideo-Val | 98.5 AP | 97.93 (official code) | 99.14 | ✅ reproduced (D3 protocol; preprocessing matched to the official code) |
+| D3 (video) | GenVideo-Val | 98.5 AP | 97.93 (official code) | 98.95 | ✅ reproduced (D3 protocol; preprocessing matched to the official code) |
 | ReStraV (video) | GenVidBench | 93.0 | — | 92.70 | ✅ match |
 | STALL (video) | GenVideo-Val | ~80 AUC | — | 83.5 AUROC | ✅ match |
 
@@ -185,7 +185,7 @@ Fidelity: SynthScope's class == the official `OmniAID` for all three releases (s
 ### Video detectors (no official weights released)
 | Method · Benchmark | Paper | SynthScope_train |
 |---|---:|---:|
-| D3 · GenVideo-Val-10 | 98.46 | 99.14 (96.15); official code 97.93 |
+| D3 · GenVideo-Val-10 | 98.46 | 98.95 (95.68); official code 97.93 |
 | D3 · EvalCrafter-14 | 98.87 | 99.30 (96.59); official code 98.70 |
 | D3 · VideoPhy (8 of 10 generators online) | 99.16 | 99.45 (95.19); official code 98.84 |
 | D3 · VidProM (4 of 6 generators, 1000-clip subsample) | 88.46 (84.95 on those 4) | 92.15 (84.86); official code 84.91 |

repro/bench_results/d3_genvideo.json

@@ -6,10 +6,20 @@
   "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test",
   "n_generators": 10,
   "means": {
-    "acc": 96.15,
-    "map": 92.15,
-    "map_real": 99.14,
-    "mauroc": 98.46
+    "acc": 95.68,
+    "map": 91.6,
+    "map_real": 98.95,
+    "mauroc": 98.23,
+    "fake_recall_at_fpr": {
+      "0.1": 0.0,
+      "1": 33.77,
+      "5": 96.71
+    },
+    "overall_fake_recall_at_fpr": {
+      "0.1": 0.0,
+      "1": 35.2,
+      "5": 96.34
+    }
   },
   "per_generator": {
     "Crafter": {
@@ -18,7 +28,12 @@
       "auroc": 98.69,
       "real_acc": 94.3,
       "fake_acc": 98.8,
-      "acc": 96.55
+      "acc": 96.55,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 44.6,
+        "5": 98.5
+      }
     },
     "Gen2": {
       "ap": 97.01,
@@ -26,7 +41,12 @@
       "auroc": 98.99,
       "real_acc": 94.3,
       "fake_acc": 100.0,
-      "acc": 97.15
+      "acc": 97.15,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 56.4,
+        "5": 100.0
+      }
     },
     "HotShot": {
       "ap": 93.95,
@@ -34,7 +54,12 @@
       "auroc": 98.42,
       "real_acc": 94.3,
       "fake_acc": 98.71,
-      "acc": 96.51
+      "acc": 96.51,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 23.29,
+        "5": 98.43
+      }
     },
     "Lavie": {
       "ap": 96.21,
@@ -42,7 +67,12 @@
       "auroc": 98.32,
       "real_acc": 94.3,
       "fake_acc": 97.4,
-      "acc": 95.85
+      "acc": 95.85,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 36.3,
+        "5": 96.7
+      }
     },
     "ModelScope": {
       "ap": 92.89,
@@ -50,7 +80,12 @@
       "auroc": 97.7,
       "real_acc": 94.3,
       "fake_acc": 94.71,
-      "acc": 94.51
+      "acc": 94.51,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 19.57,
+        "5": 93.86
+      }
     },
     "MoonValley": {
       "ap": 96.08,
@@ -58,7 +93,12 @@
       "auroc": 98.98,
       "real_acc": 94.3,
       "fake_acc": 99.84,
-      "acc": 97.07
+      "acc": 97.07,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 59.58,
+        "5": 99.84
+      }
     },
     "MorphStudio": {
       "ap": 94.1,
@@ -66,7 +106,12 @@
       "auroc": 98.49,
       "real_acc": 94.3,
       "fake_acc": 99.14,
-      "acc": 96.72
+      "acc": 96.72,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 25.14,
+        "5": 99.0
+      }
     },
     "Show_1": {
       "ap": 94.62,
@@ -74,7 +119,12 @@
       "auroc": 98.72,
       "real_acc": 94.3,
       "fake_acc": 99.57,
-      "acc": 96.94
+      "acc": 96.94,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 38.71,
+        "5": 99.57
+      }
     },
     "Sora": {
       "ap": 62.39,
@@ -82,17 +132,27 @@
       "auroc": 98.45,
       "real_acc": 94.3,
       "fake_acc": 98.21,
-      "acc": 96.26
+      "acc": 96.26,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 23.21,
+        "5": 98.21
+      }
     },
     "WildScrape": {
-      "ap": 97.29,
-      "ap_real": 98.21,
-      "auroc": 97.83,
+      "ap": 91.88,
+      "ap_real": 96.33,
+      "auroc": 95.55,
       "real_acc": 94.3,
-      "fake_acc": 93.51,
-      "acc": 93.9
+      "fake_acc": 84.09,
+      "acc": 89.2,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 10.93,
+        "5": 83.01
+      }
     }
   },
-  "date": "2026-10-11T18:03:10",
+  "date": "2026-11-01T00:56:39",
   "synthscope_version": "video"
 }

repro/bench_results/d3_genvideo_fpr.json

@@ -0,0 +1,158 @@
+{
+  "tag": "d3_genvideo_fpr",
+  "model": "d3",
+  "ckpt": "registry-default",
+  "benchmark": "video:pergen",
+  "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test",
+  "n_generators": 10,
+  "means": {
+    "acc": 95.53,
+    "map": 68.84,
+    "map_real": 99.87,
+    "mauroc": 98.45,
+    "fake_recall_at_fpr": {
+      "0.1": 0.21,
+      "1": 62.03,
+      "5": 96.52
+    },
+    "overall_fake_recall_at_fpr": {
+      "0.1": 0.24,
+      "1": 63.32,
+      "5": 96.41
+    }
+  },
+  "per_generator": {
+    "Crafter": {
+      "ap": 84.96,
+      "ap_real": 99.86,
+      "auroc": 98.9,
+      "real_acc": 93.99,
+      "fake_acc": 98.93,
+      "acc": 96.46,
+      "fake_recall_at_fpr": {
+        "0.1": 0.57,
+        "1": 69.24,
+        "5": 98.57
+      }
+    },
+    "Gen2": {
+      "ap": 85.37,
+      "ap_real": 99.9,
+      "auroc": 99.17,
+      "real_acc": 93.99,
+      "fake_acc": 100.0,
+      "acc": 96.99,
+      "fake_recall_at_fpr": {
+        "0.1": 0.07,
+        "1": 84.28,
+        "5": 100.0
+      }
+    },
+    "HotShot": {
+      "ap": 69.37,
+      "ap_real": 99.91,
+      "auroc": 98.65,
+      "real_acc": 93.99,
+      "fake_acc": 98.71,
+      "acc": 96.35,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 56.29,
+        "5": 98.29
+      }
+    },
+    "Lavie": {
+      "ap": 82.22,
+      "ap_real": 99.79,
+      "auroc": 98.52,
+      "real_acc": 93.99,
+      "fake_acc": 97.43,
+      "acc": 95.71,
+      "fake_recall_at_fpr": {
+        "0.1": 0.36,
+        "1": 62.36,
+        "5": 96.29
+      }
+    },
+    "ModelScope": {
+      "ap": 66.32,
+      "ap_real": 99.85,
+      "auroc": 97.93,
+      "real_acc": 93.99,
+      "fake_acc": 94.71,
+      "acc": 94.35,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 48.57,
+        "5": 93.86
+      }
+    },
+    "MoonValley": {
+      "ap": 76.11,
+      "ap_real": 99.95,
+      "auroc": 99.15,
+      "real_acc": 93.99,
+      "fake_acc": 99.84,
+      "acc": 96.92,
+      "fake_recall_at_fpr": {
+        "0.1": 1.12,
+        "1": 81.95,
+        "5": 99.84
+      }
+    },
+    "MorphStudio": {
+      "ap": 69.42,
+      "ap_real": 99.91,
+      "auroc": 98.71,
+      "real_acc": 93.99,
+      "fake_acc": 99.14,
+      "acc": 96.57,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 51.57,
+        "5": 98.57
+      }
+    },
+    "Show_1": {
+      "ap": 72.83,
+      "ap_real": 99.93,
+      "auroc": 98.93,
+      "real_acc": 93.99,
+      "fake_acc": 99.57,
+      "acc": 96.78,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 68.29,
+        "5": 99.57
+      }
+    },
+    "Sora": {
+      "ap": 19.13,
+      "ap_real": 99.99,
+      "auroc": 98.69,
+      "real_acc": 93.99,
+      "fake_acc": 98.21,
+      "acc": 96.1,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 64.29,
+        "5": 98.21
+      }
+    },
+    "WildScrape": {
+      "ap": 62.68,
+      "ap_real": 99.58,
+      "auroc": 95.82,
+      "real_acc": 93.99,
+      "fake_acc": 84.09,
+      "acc": 89.04,
+      "fake_recall_at_fpr": {
+        "0.1": 0.0,
+        "1": 33.44,
+        "5": 82.03
+      }
+    }
+  },
+  "date": "2026-10-31T23:42:33",
+  "synthscope_version": "video"
+}

repro/bench_results/split_genvideo_fpr.json

@@ -0,0 +1,158 @@
+{
+  "tag": "split_genvideo_fpr",
+  "model": "split",
+  "ckpt": "registry-default",
+  "benchmark": "video:pergen",
+  "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test",
+  "n_generators": 10,
+  "means": {
+    "acc": 99.41,
+    "map": 99.79,
+    "map_real": 100.0,
+    "mauroc": 99.97,
+    "fake_recall_at_fpr": {
+      "0.1": 98.94,
+      "1": 99.87,
+      "5": 99.98
+    },
+    "overall_fake_recall_at_fpr": {
+      "0.1": 98.87,
+      "1": 99.86,
+      "5": 99.98
+    }
+  },
+  "per_generator": {
+    "Crafter": {
+      "ap": 99.98,
+      "ap_real": 100.0,
+      "auroc": 100.0,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "Gen2": {
+      "ap": 99.84,
+      "ap_real": 100.0,
+      "auroc": 99.98,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "HotShot": {
+      "ap": 100.0,
+      "ap_real": 100.0,
+      "auroc": 100.0,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "Lavie": {
+      "ap": 99.96,
+      "ap_real": 100.0,
+      "auroc": 99.99,
+      "real_acc": 98.87,
+      "fake_acc": 99.93,
+      "acc": 99.4,
+      "fake_recall_at_fpr": {
+        "0.1": 99.93,
+        "1": 99.93,
+        "5": 100.0
+      }
+    },
+    "ModelScope": {
+      "ap": 99.89,
+      "ap_real": 100.0,
+      "auroc": 99.99,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 99.57,
+        "1": 99.86,
+        "5": 100.0
+      }
+    },
+    "MoonValley": {
+      "ap": 99.62,
+      "ap_real": 100.0,
+      "auroc": 99.98,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "MorphStudio": {
+      "ap": 99.97,
+      "ap_real": 100.0,
+      "auroc": 100.0,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "Show_1": {
+      "ap": 99.96,
+      "ap_real": 100.0,
+      "auroc": 100.0,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "Sora": {
+      "ap": 99.63,
+      "ap_real": 100.0,
+      "auroc": 100.0,
+      "real_acc": 98.87,
+      "fake_acc": 100.0,
+      "acc": 99.43,
+      "fake_recall_at_fpr": {
+        "0.1": 100.0,
+        "1": 100.0,
+        "5": 100.0
+      }
+    },
+    "WildScrape": {
+      "ap": 99.09,
+      "ap_real": 99.97,
+      "auroc": 99.79,
+      "real_acc": 98.87,
+      "fake_acc": 99.57,
+      "acc": 99.22,
+      "fake_recall_at_fpr": {
+        "0.1": 89.94,
+        "1": 98.92,
+        "5": 99.78
+      }
+    }
+  },
+  "date": "2026-10-31T22:35:53",
+  "synthscope_version": "video"
+}

repro/bench_results/split_official_genvideo.json

@@ -0,0 +1,156 @@
+{
+  "tag": "split_official_genvideo",
+  "model": "split",
+  "ckpt": "official SPLIT code (XCLIP-16), training-free",
+  "benchmark": "video:official-split",
+  "data_root": "/ssdstore/ssalehi/SynthScope/GenVideo/test",
+  "n_generators": 10,
+  "means": {
+    "overall_fake_recall_at_fpr": {
+      "0.1": 82.22,
+      "1": 90.17,
+      "5": 96.89
+    },
+    "mean_fake_recall_at_fpr": {
+      "0.1": 83.66,
+      "1": 91.21,
+      "5": 97.23
+    },
+    "map": 96.12
+  },
+  "overall": {
+    "fake_recall_at_fpr": {
+      "0.1": 82.22,
+      "1": 90.17,
+      "5": 96.89
+    },
+    "ap": 99.19,
+    "ap_real": 98.99,
+    "auroc": 99.08,
+    "n_real": 10000,
+    "n_fake": 8573
+  },
+  "per_generator": {
+    "Crafter": {
+      "fake_recall_at_fpr": {
+        "0.1": 89.99,
+        "1": 96.21,
+        "5": 99.64
+      },
+      "ap": 99.25,
+      "ap_real": 99.98,
+      "auroc": 99.88,
+      "n_real": 10000,
+      "n_fake": 1398
+    },
+    "Gen2": {
+      "fake_recall_at_fpr": {
+        "0.1": 56.16,
+        "1": 76.3,
+        "5": 94.64
+      },
+      "ap": 94.42,
+      "ap_real": 99.88,
+      "auroc": 99.1,
+      "n_real": 10000,
+      "n_fake": 1380
+    },
+    "HotShot": {
+      "fake_recall_at_fpr": {
+        "0.1": 99.86,
+        "1": 100.0,
+        "5": 100.0
+      },
+      "ap": 100.0,
+      "ap_real": 100.0,
+      "auroc": 100.0,
+      "n_real": 10000,
+      "n_fake": 700
+    },
+    "Lavie": {
+      "fake_recall_at_fpr": {
+        "0.1": 93.21,
+        "1": 96.07,
+        "5": 98.71
+      },
+      "ap": 98.94,
+      "ap_real": 99.96,
+      "auroc": 99.73,
+      "n_real": 10000,
+      "n_fake": 1400
+    },
+    "ModelScope": {
+      "fake_recall_at_fpr": {
+        "0.1": 87.29,
+        "1": 93.14,
+        "5": 97.86
+      },
+      "ap": 97.2,
+      "ap_real": 99.97,
+      "auroc": 99.58,
+      "n_real": 10000,
+      "n_fake": 700
+    },
+    "MoonValley": {
+      "fake_recall_at_fpr": {
+        "0.1": 52.88,
+        "1": 74.76,
+        "5": 95.37
+      },
+      "ap": 89.71,
+      "ap_real": 99.94,
+      "auroc": 99.11,
+      "n_real": 10000,
+      "n_fake": 626
+    },
+    "MorphStudio": {
+      "fake_recall_at_fpr": {
+        "0.1": 95.0,
+        "1": 98.0,
+        "5": 99.71
+      },
+      "ap": 99.39,
+      "ap_real": 100.0,
+      "auroc": 99.94,
+      "n_real": 10000,
+      "n_fake": 700
+    },
+    "Show_1": {
+      "fake_recall_at_fpr": {
+        "0.1": 97.86,
+        "1": 99.71,
+        "5": 100.0
+      },
+      "ap": 99.85,
+      "ap_real": 100.0,
+      "auroc": 99.99,
+      "n_real": 10000,
+      "n_fake": 700
+    },
+    "Sora": {
+      "fake_recall_at_fpr": {
+        "0.1": 91.07,
+        "1": 98.21,
+        "5": 100.0
+      },
+      "ap": 94.72,
+      "ap_real": 100.0,
+      "auroc": 99.91,
+      "n_real": 10000,
+      "n_fake": 56
+    },
+    "WildScrape": {
+      "fake_recall_at_fpr": {
+        "0.1": 73.27,
+        "1": 79.74,
+        "5": 86.42
+      },
+      "ap": 87.71,
+      "ap_real": 99.26,
+      "auroc": 94.31,
+      "n_real": 10000,
+      "n_fake": 913
+    }
+  },
+  "date": "2026-10-31T19:33:09"
+}

repro/benchmark_specs/d3.json

@@ -3,7 +3,7 @@
   "paper": "D3: Training-Free AI-Generated Video Detection Using Second-Order Features (Zheng et al., ICCV 2025)",
   "arxiv_id": "n/a (CVF Open Access ICCV 2025; code github.com/Zig-HS/D3)",
   "official_repo": "https://github.com/Zig-HS/D3",
-  "training_set_in_paper": "NONE \u2014 D3 is training-free (inference only). Baselines (re-implemented for comparison) were trained on real videos from Youku-mPLUG + Pika fakes, following DeMamba (paper Sec 4.1).",
+  "training_set_in_paper": "NONE — D3 is training-free (inference only). Baselines (re-implemented for comparison) were trained on real videos from Youku-mPLUG + Pika fakes, following DeMamba (paper Sec 4.1).",
   "encoder_in_paper": "XCLIP-B/16 (X-CLIP ViT-B/16, Ni et al. ECCV'22, paper ref [34]) as the frozen per-frame visual encoder; first-order = L2 distance. CLIP-B/16 is the ablation runner-up (~97 on GenVideo, Table 7).",
   "paper_protocol": "Extract up to a 2 s segment, sample frames at 8 fps, JPEG; crop 10% of the longer edge then resize to 224x224. Score = std of 2nd-order central-difference of inter-frame L2 distances (Eq.8). Metric reported: AP (real-class); AUROC in supplementary. Hardware RTX 4090. (paper Sec 4.1 'Implementation Details')",
   "synthscope_encoder": "X-CLIP-B/16 (microsoft/xclip-base-patch16), as in the paper. SynthScope's D3 gives the same score as the official D3_model on the same frames (max |diff| 0, repro check 2026-10-11).",
@@ -45,11 +45,11 @@
       "official_eval_protocol": "XCLIP-B/16 frozen, 8 fps over <=2 s, JPEG, 10% longer-edge crop, resize 224x224, L2 first-order, std of 2nd-order central diff; AP computed real-class (positive=real, score=+std), max_len=1000 per class (official eval.py).",
       "our_status": "reproduced",
       "our_number": {
-        "map": 99.14,
-        "auroc": 98.46,
-        "acc": 96.15,
+        "map": 98.95,
+        "auroc": 98.23,
+        "acc": 95.68,
         "official_code_map": 97.93,
-        "note": "D3 protocol (--limit 1000: first 1000 clips per class per generator; AP with real as the positive class). SynthScope engine 99.14 AP (AUROC 98.46, balanced acc 96.15 at the refitted calibration; tag d3_genvideo, job 252246). Official D3 code end to end: 97.93 (tag d3_official_genvideo, job 252271): 9 of 10 generators within 0.25 of the paper; WildScrape 92.50 vs 96.49 (282 GIFs whose length decord cannot read start at 0 instead of a random offset). The old 63.93 compared 10,000 reals with as few as 56 fakes and used AP with fake as positive."
+        "note": "D3 protocol (--limit 1000: first 1000 clips per class per generator; AP with real as the positive class). SynthScope engine 98.95 AP (AUROC 98.23, balanced acc 95.68 at the refitted calibration; tag d3_genvideo, job 270428, after the GIF decode fix: WildScrape 96.33 vs paper 96.49, was 98.21 when GIFs decoded as black clips). Official D3 code end to end: 97.93 (tag d3_official_genvideo, job 252271): 9 of 10 generators within 0.25 of the paper; WildScrape 92.50 vs 96.49 (282 GIFs whose length decord cannot read start at 0 instead of a random offset). The old 63.93 compared 10,000 reals with as few as 56 fakes and used AP with fake as positive."
       },
       "our_per_generator_ap": {
         "Crafter": 99.14,
@@ -61,10 +61,10 @@
         "MorphStudio": 99.27,
         "Show_1": 99.37,
         "Sora": 99.91,
-        "WildScrape": 98.21
+        "WildScrape": 96.33
       },
       "dataset_source": "on-disk: /hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val (14 GB; downloaded from ModelScope cccnju/Gen-Video GenVideo-Val.zip). The real source is MSR-VTT.",
-      "xclip_weights_downloadable": "YES \u2014 microsoft/xclip-base-patch16 on HuggingFace (Kinetics-400 trained, NOT gated). Matches paper ref [34]. The gap-closing fix (swap CLIP->XCLIP) is unblocked by available weights.",
+      "xclip_weights_downloadable": "YES — microsoft/xclip-base-patch16 on HuggingFace (Kinetics-400 trained, NOT gated). Matches paper ref [34]. The gap-closing fix (swap CLIP->XCLIP) is unblocked by available weights.",
       "official_code_per_generator_ap": {
         "Crafter": 98.28,
         "Gen2": 99.24,
@@ -281,4 +281,4 @@
     "n_benchmarks_reported": 4,
     "key_finding": "Since 2026-10-11 SynthScope's D3 follows the official preprocessing (16 frames at 8 fps, 8 for clips under 2 s; BGR; 10% crop of the longer side; ImageNet normalization) and is scored with D3's protocol (first 1000 clips per class, AP with real as the positive class). The official D3 code run end to end on the same data (repro/official_evals/d3_official_eval.py) reproduces GenVideo 97.93, EvalCrafter 98.70, VideoPhy 98.84 (8 of 10 generators) and VidProM 84.91 (4 of 6 generators, paper 84.95 on those 4); SynthScope's engine gives 99.14 / 99.30 / 99.45 / 92.15. The remaining engine-vs-official difference is only the official JPEG frame round-trip."
   }
-}
+}

repro/benchmark_specs/split.json

@@ -0,0 +1,233 @@
+{
+  "model": "split",
+  "paper": "SPLIT: Training-Free AI-Generated and Partially Edited Video Detection via Spatial Patch-Level Incoherence and Temporal Roughness (Hyun & Kim, ECCV 2026)",
+  "arxiv_id": "2607.02886",
+  "official_repo": "https://github.com/mldljyh/SPLIT",
+  "training_set_in_paper": "NONE - SPLIT is training-free (frozen encoder, real-only threshold calibration).",
+  "encoder_in_paper": "X-CLIP B/16 (microsoft/xclip-base-patch16) patch tokens; CLIP, DINO and CNN encoders are ablations.",
+  "paper_protocol": "Official video2frame.py: ffmpeg JPEG frames, 8 fps over 3 s (random start for longer clips); read_video: 16 frames (8 if fewer than 16), cv2 BGR, 10% crop of the longer side, resize 224, ImageNet normalization. Score = TTR**8 * LSMI. Metric: fake recall at FPR 0.1/1/5%, threshold = (1-FPR) quantile of the real scores (eval.py evaluate_split).",
+  "synthscope_encoder": "X-CLIP B/16, as in the paper. SynthScope's own implementation (the official repo has no license) returns the official SPLIT_model.forward_score on the same frames (relative difference <= 7e-6, job 269966).",
+  "benchmarks": [
+    {
+      "name": "GenVideo (GenVideo-Val, MSR-VTT real calibration, supp. Table 18)",
+      "datasets": [
+        "GenVideo-Val"
+      ],
+      "subsets_or_generators": [
+        "Crafter",
+        "Gen2",
+        "HotShot",
+        "Lavie",
+        "ModelScope",
+        "MoonValley",
+        "MorphStudio",
+        "Show_1",
+        "Sora",
+        "WildScrape"
+      ],
+      "metric": "fake recall @ FPR 0.1/1/5% (Overall = all fakes pooled)",
+      "paper_number": {
+        "fake_recall_fpr0.1": 82.13,
+        "fake_recall_fpr1": 91.69,
+        "fake_recall_fpr5": 97.2
+      },
+      "paper_per_generator": {
+        "Crafter": {
+          "fpr_0.1": 88.84,
+          "fpr_1": 96.78,
+          "fpr_5": 99.64
+        },
+        "Gen2": {
+          "fpr_0.1": 53.26,
+          "fpr_1": 77.61,
+          "fpr_5": 94.71
+        },
+        "HotShot": {
+          "fpr_0.1": 99.86,
+          "fpr_1": 100,
+          "fpr_5": 100
+        },
+        "Lavie": {
+          "fpr_0.1": 92.5,
+          "fpr_1": 96.36,
+          "fpr_5": 98.79
+        },
+        "ModelScope": {
+          "fpr_0.1": 86.71,
+          "fpr_1": 93.57,
+          "fpr_5": 97.86
+        },
+        "MoonValley": {
+          "fpr_0.1": 49.36,
+          "fpr_1": 76.36,
+          "fpr_5": 95.37
+        },
+        "MorphStudio": {
+          "fpr_0.1": 94.71,
+          "fpr_1": 98.43,
+          "fpr_5": 99.71
+        },
+        "Show_1": {
+          "fpr_0.1": 96.86,
+          "fpr_1": 99.86,
+          "fpr_5": 100
+        },
+        "Sora": {
+          "fpr_0.1": 87.5,
+          "fpr_1": 98.21,
+          "fpr_5": 100
+        },
+        "WildScrape": {
+          "fpr_0.1": 71.66,
+          "fpr_1": 79.76,
+          "fpr_5": 85.89
+        }
+      },
+      "official_eval_protocol": "All 10,000 MSR-VTT reals calibrate the threshold; all fakes of the 10 generators scored (official eval.py, no max_len).",
+      "our_status": "reproduced",
+      "our_number": {
+        "official_code_our_data": {
+          "fpr_0.1": 82.22,
+          "fpr_1": 90.17,
+          "fpr_5": 96.89
+        },
+        "synthscope": {
+          "fpr_0.1": 98.87,
+          "fpr_1": 99.86,
+          "fpr_5": 99.98
+        },
+        "note": "Official code on our data (JPEG frame round-trip, job 270323) reproduces the paper: Overall fake recall 82.22/90.17/96.89 vs 82.13/91.69/97.20. SynthScope decodes frames directly (no JPEG) and reaches 98.87/99.86/99.98 (job 270384; mAP 99.79, AUROC 99.97, acc 99.41 at the calibrated threshold)."
+      },
+      "official_code_per_generator": {
+        "Crafter": {
+          "fpr_0.1": 89.99,
+          "fpr_1": 96.21,
+          "fpr_5": 99.64
+        },
+        "Gen2": {
+          "fpr_0.1": 56.16,
+          "fpr_1": 76.3,
+          "fpr_5": 94.64
+        },
+        "HotShot": {
+          "fpr_0.1": 99.86,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "Lavie": {
+          "fpr_0.1": 93.21,
+          "fpr_1": 96.07,
+          "fpr_5": 98.71
+        },
+        "ModelScope": {
+          "fpr_0.1": 87.29,
+          "fpr_1": 93.14,
+          "fpr_5": 97.86
+        },
+        "MoonValley": {
+          "fpr_0.1": 52.88,
+          "fpr_1": 74.76,
+          "fpr_5": 95.37
+        },
+        "MorphStudio": {
+          "fpr_0.1": 95.0,
+          "fpr_1": 98.0,
+          "fpr_5": 99.71
+        },
+        "Show_1": {
+          "fpr_0.1": 97.86,
+          "fpr_1": 99.71,
+          "fpr_5": 100.0
+        },
+        "Sora": {
+          "fpr_0.1": 91.07,
+          "fpr_1": 98.21,
+          "fpr_5": 100.0
+        },
+        "WildScrape": {
+          "fpr_0.1": 73.27,
+          "fpr_1": 79.74,
+          "fpr_5": 86.42
+        }
+      },
+      "our_per_generator": {
+        "Crafter": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "Gen2": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "HotShot": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "Lavie": {
+          "fpr_0.1": 99.93,
+          "fpr_1": 99.93,
+          "fpr_5": 100.0
+        },
+        "ModelScope": {
+          "fpr_0.1": 99.57,
+          "fpr_1": 99.86,
+          "fpr_5": 100.0
+        },
+        "MoonValley": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "MorphStudio": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "Show_1": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "Sora": {
+          "fpr_0.1": 100.0,
+          "fpr_1": 100.0,
+          "fpr_5": 100.0
+        },
+        "WildScrape": {
+          "fpr_0.1": 89.94,
+          "fpr_1": 98.92,
+          "fpr_5": 99.78
+        }
+      },
+      "dataset_source": "genvideo loader (ModelScope cccnju/Gen-Video GenVideo-Val.zip); reals = MSR-VTT."
+    },
+    {
+      "name": "FakeParts (partial edits; Tables 1, 15-16)",
+      "datasets": [
+        "FakeParts"
+      ],
+      "metric": "fake recall @ FPR",
+      "paper_number": {
+        "fake_recall_fpr0.1": 74.45
+      },
+      "our_status": "missing",
+      "dataset_source": "not in SynthScope yet"
+    },
+    {
+      "name": "ViF-Bench (Tables 3, 19)",
+      "datasets": [
+        "ViF-Bench"
+      ],
+      "metric": "fake recall @ FPR",
+      "paper_number": {
+        "fake_recall_fpr0.1": 84.52
+      },
+      "our_status": "missing",
+      "dataset_source": "not in SynthScope yet"
+    }
+  ],
+  "summary": "Training-free; exact parity with the official model. The official pipeline on our GenVideo copy reproduces the paper's MSR-VTT-calibrated table; SynthScope's direct decoding (no JPEG round-trip) is much stronger at low FPR."
+}

repro/official_evals/split_official_eval.py

@@ -0,0 +1,139 @@
+"""Run the OFFICIAL SPLIT pipeline (Hyun & Kim, ECCV 2026; github.com/mldljyh/SPLIT) on a SynthScope video dataset.
+
+SynthScope's SPLIT gives the official ``SPLIT_model`` scores on the same frames (relative difference ~1e-6), but the
+official numbers also depend on the official frame extraction, which SPLIT shares with D3: ``utils/video2frame.py``
+writes JPEG frames with ffmpeg (8 fps over 3 s, random start for longer clips), ``data/datasets.py`` reads them back
+with ``cv2.imread`` (BGR), crops 10% of the longer side, resizes to 224 and normalizes with ImageNet statistics. The
+frames are extracted exactly as ``d3_official_eval.py`` does (same seeded start, same frame folders, so the two share a
+cache), and the metrics come from the official ``eval.py``'s ``evaluate_split`` unchanged: fake recall at FPR 0.1 / 1 /
+5 % with the threshold set on the real scores alone (``np.quantile(real, 1 - fpr, method="higher")``).
+
+Like the paper's GenVideo table, every real clip (GenVideo-Val's MSR-VTT reals) calibrates the threshold and every fake
+clip is scored, per generator and pooled over all generators ("Overall"); there is no per-class limit unless ``--limit``
+is given.
+
+    python repro/official_evals/split_official_eval.py --dataset-root /data/GenVideo/test --name genvideo \\
+        --official-repo /path/to/SPLIT --albumentations-path /path/to/site-packages --ffmpeg /path/to/ffmpeg \\
+        --frames /scratch/d3_frames/genvideo
+
+Output: repro/bench_results/<tag>.json and a line in repro/MASTER_RESULTS.jsonl.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import hashlib
+import importlib.util
+import json
+import random
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from d3_official_eval import extract, videos  # noqa: E402  (the shared official frame extraction)
+
+REPRO = Path(__file__).resolve().parents[1]
+RESULTS_LOG = REPRO / "MASTER_RESULTS.jsonl"
+FPRS = (0.001, 0.01, 0.05)
+
+
+def load_official_eval(repo: str):
+    """The official ``eval.py`` as a module (its ``evaluate_split`` computes the paper's metrics)."""
+    spec = importlib.util.spec_from_file_location("split_official_eval_py", Path(repo) / "eval.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@torch.no_grad()
+def scores(model, read_video, trans, paths, frames_root: Path, ffmpeg: str, device) -> list[float]:
+    out = []
+    for p in paths:
+        folder = extract(p, frames_root / hashlib.md5(str(p.resolve()).encode()).hexdigest()[:16], ffmpeg)
+        try:
+            x = read_video(str(folder), trans=trans).unsqueeze(0).to(device)    # official reader: needs >= 8 frames
+        except Exception as e:  # noqa: BLE001
+            print(f"    skip {p.name}: {e}")
+            continue
+        out.append(float(model.forward_score(x).item()))
+    return out
+
+
+def summarize(official, real: list[float], fake: list[float]) -> dict:
+    m = official.evaluate_split(np.array(real), np.array(fake))
+    return {"fake_recall_at_fpr": {f"{a * 100:g}": round(m["fake_recall_at_fpr"][a] * 100, 2) for a in FPRS},
+            "ap": round(m["ap_fake"] * 100, 2), "ap_real": round(m["ap_real"] * 100, 2),
+            "auroc": round(m["roc_auc"] * 100, 2), "n_real": len(real), "n_fake": len(fake)}
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--dataset-root", required=True, help="<generator>/{0_real,1_fake} tree")
+    ap.add_argument("--name", required=True, help="dataset name for the tag, e.g. genvideo")
+    ap.add_argument("--official-repo", required=True, help="clone of github.com/mldljyh/SPLIT")
+    ap.add_argument("--albumentations-path", default=None, help="extra sys.path entry providing albumentations")
+    ap.add_argument("--ffmpeg", default="ffmpeg")
+    ap.add_argument("--frames", required=True, help="scratch folder for the extracted JPEG frames")
+    ap.add_argument("--limit", type=int, default=0, help="clips per class per generator (0 = all)")
+    ap.add_argument("--encoder", default="XCLIP-16")
+    ap.add_argument("--select", default=None, help="comma list: only these generators")
+    ap.add_argument("--tag", default=None)
+    args = ap.parse_args(argv)
+
+    sys.path[:0] = [args.official_repo] + ([args.albumentations_path] if args.albumentations_path else [])
+    from data.datasets import read_video, set_preprocessing      # official SPLIT
+    from models.SPLIT_model import SPLIT_model                   # official SPLIT
+    official = load_official_eval(args.official_repo)
+
+    random.seed(42); np.random.seed(42); torch.manual_seed(42)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = SPLIT_model(encoder_type=args.encoder, loss_type="l2").to(device).eval()
+    trans = set_preprocessing(None, None)
+    root, frames_root = Path(args.dataset_root), Path(args.frames)
+    tag = args.tag or f"split_official_{args.name}"
+    cap = (lambda xs: xs[:args.limit]) if args.limit else (lambda xs: xs)
+    gens = sorted(p for p in root.iterdir() if p.is_dir()
+                  and (not args.select or p.name in args.select.split(",")))
+    print(f"=== split_official_eval {tag} | {len(gens)} generators | {args.encoder} | device={device} ===", flush=True)
+
+    cache: dict = {}
+    per_gen, all_real, all_fake = {}, None, []
+    for g in gens:
+        reals, fakes = cap(videos(g / "0_real")), cap(videos(g / "1_fake"))
+        if not reals or not fakes:
+            continue
+        key = tuple(p.name for p in reals)     # GenVideo copies the same MSR-VTT reals into every generator folder
+        if key not in cache:
+            cache[key] = scores(model, read_video, trans, reals, frames_root, args.ffmpeg, device)
+        rs = cache[key]
+        fs = scores(model, read_video, trans, fakes, frames_root, args.ffmpeg, device)
+        per_gen[g.name] = summarize(official, rs, fs)
+        all_real = rs if all_real is None or len(rs) > len(all_real) else all_real
+        all_fake += fs
+        print(f"  {g.name}: {len(rs)} real + {len(fs)} fake  fake recall @ FPR 0.1/1/5% = "
+              f"{list(per_gen[g.name]['fake_recall_at_fpr'].values())}  AP={per_gen[g.name]['ap']}", flush=True)
+
+    overall = summarize(official, all_real, all_fake)
+    mean = lambda f: round(sum(v["fake_recall_at_fpr"][f] for v in per_gen.values()) / len(per_gen), 2)  # noqa: E731
+    means = {"overall_fake_recall_at_fpr": overall["fake_recall_at_fpr"],
+             "mean_fake_recall_at_fpr": {f: mean(f) for f in overall["fake_recall_at_fpr"]},
+             "map": round(sum(v["ap"] for v in per_gen.values()) / len(per_gen), 2)}
+    result = {"tag": tag, "model": "split", "ckpt": f"official SPLIT code ({args.encoder}), training-free",
+              "benchmark": "video:official-split", "data_root": str(root), "n_generators": len(per_gen),
+              "means": means, "overall": overall, "per_generator": per_gen,
+              "date": datetime.datetime.now().isoformat(timespec="seconds")}
+    out = REPRO / "bench_results" / f"{tag}.json"
+    out.write_text(json.dumps(result, indent=2) + "\n")
+    with RESULTS_LOG.open("a") as f:
+        f.write(json.dumps({k: result[k] for k in ("date", "tag", "model", "ckpt", "benchmark", "data_root",
+                                                   "n_generators", "means")}) + "\n")
+    print(f"\n=== RESULT {tag}: Overall fake recall @ FPR 0.1/1/5% = {list(overall['fake_recall_at_fpr'].values())} "
+          f"over {len(per_gen)} generators -> {out} ===", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_split_smoke.py

@@ -0,0 +1,92 @@
+"""Smoke test for the SPLIT integration (no weight downloads: a tiny randomly initialized X-CLIP vision tower stands in
+for X-CLIP B/16, saved to a temporary directory and loaded through the model's normal ``from_pretrained`` path).
+
+Checks, on CPU:
+* temporal roughness: a straight constant-speed token trajectory gives TTR = 0, a back-and-forth (jittery) one gives
+  TTR > 0, matching the paper's definition log2(L1 / L2),
+* motion incoherence: a motion field that is the same for every patch gives 0, a varying one gives > 0,
+* the registry entry is training-free and uses D3's video preprocessing (16 frames, 224, BGR, ImageNet normalization),
+* ``forward`` -> [B, 2] logits whose margin is ``score - threshold``; ``compute_loss`` returns a zero loss that
+  backpropagates (the trainer runs, nothing is learned) and the encoder stays frozen in train mode.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+
+import torch
+from omegaconf import OmegaConf
+from transformers import XCLIPVisionConfig, XCLIPVisionModel
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "interface"))
+
+from models.video.split import split as split_module  # noqa: E402
+from models.video.split.split import SynthScopeSPLITModel  # noqa: E402
+from backend.registry import get_spec                     # noqa: E402
+
+TINY = XCLIPVisionConfig(hidden_size=32, intermediate_size=64, num_hidden_layers=2, num_attention_heads=2,
+                         image_size=64, patch_size=16, mit_hidden_size=32, mit_intermediate_size=64,
+                         mit_num_hidden_layers=1, mit_num_attention_heads=2, num_frames=4)
+
+
+def build_model(tmp: str) -> SynthScopeSPLITModel:
+    XCLIPVisionModel(TINY).save_pretrained(tmp)
+    split_module.ENCODERS["tiny"] = tmp
+    cfg = OmegaConf.create({"ModelConfig": {"model_type": "split", "encoder": "tiny", "gamma": 8.0, "threshold": 0.5},
+                            "TrainerConfig": {}})
+    return SynthScopeSPLITModel(cfg)
+
+
+def test_temporal_roughness() -> None:
+    print("[A] temporal roughness ...")
+    t, n, d = 8, 4, 16
+    direction = torch.randn(n, d)
+    smooth = torch.arange(t, dtype=torch.float32)[None, :, None, None] * direction[None, None]   # [1, T, N, D]
+    ttr = SynthScopeSPLITModel.temporal_roughness(smooth)
+    assert ttr.shape == (1,) and abs(float(ttr)) < 1e-4, float(ttr)
+    jitter = smooth.clone()
+    jitter[:, 1::2] += 5 * torch.randn(n, d)          # odd frames jump away and back: long 1-step, short 2-step paths
+    assert float(SynthScopeSPLITModel.temporal_roughness(jitter)) > 0.5
+
+
+def test_motion_incoherence() -> None:
+    print("[B] motion incoherence ...")
+    t, g, d = 5, 3, 8
+    uniform = torch.arange(t, dtype=torch.float32)[None, :, None, None] * torch.randn(d)[None, None, None]
+    uniform = uniform.expand(1, t, g * g, d)          # every patch moves by the same vector
+    assert float(SynthScopeSPLITModel.motion_incoherence(uniform)) < 1e-5
+    assert float(SynthScopeSPLITModel.motion_incoherence(torch.randn(1, t, g * g, d))) > 0.1
+
+
+def test_registry() -> None:
+    print("[C] registry entry ...")
+    spec = get_spec("split")
+    assert spec.modality == "video" and spec.checkpoint is None and spec.weights == "training_free"
+    assert spec.num_frames == 16 and spec.frame_size == 224 and spec.bgr and spec.edge_crop == 0.1
+
+
+def test_forward_and_loss(model: SynthScopeSPLITModel) -> None:
+    print("[D] forward / compute_loss ...")
+    frames = torch.randn(2, 6, 3, 64, 64)
+    model.eval()
+    with torch.no_grad():
+        logits, score = model(frames), model.score(frames)
+    assert logits.shape == (2, 2) and torch.isfinite(logits).all() and (score >= 0).all()
+    assert torch.allclose(logits[:, 1] - logits[:, 0], 2 * (score - model.threshold))
+    model.train()
+    assert not model.encoder.training and not any(p.requires_grad for p in model.encoder.parameters())
+    out = model.compute_loss((frames, torch.tensor([0, 1]), torch.tensor([0, 0])))
+    out["loss"].backward()
+    assert out["loss"].item() == 0.0 and model._noop.grad is not None
+
+
+if __name__ == "__main__":
+    test_temporal_roughness()
+    test_motion_incoherence()
+    test_registry()
+    with tempfile.TemporaryDirectory() as d:
+        test_forward_and_loss(build_model(d))
+    print("SPLIT smoke test passed")

tests/test_video_gif_decode.py

@@ -0,0 +1,41 @@
+"""Regression test: animated GIFs decode to their real frames, not a black clip.
+
+decord cannot open GIFs ("broken metadata"), and ``sample_video_frames`` used to return an all-zero clip for them, so
+every video detector saw GIFs (e.g. 282 of GenVideo's WildScrape fakes) as black, motionless video. The PIL fallback
+must return the GIF's own frames at the requested sampling rate.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+
+import numpy as np
+from PIL import Image
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from dataset import sample_video_frames  # noqa: E402
+
+
+def make_gif(path: str, n: int = 24, duration_ms: int = 100) -> None:
+    """``n`` frames at 10 fps (GIF stores durations in 10 ms steps); frame k is a solid gray of level 10 * k."""
+    frames = [Image.fromarray(np.full((64, 96, 3), 10 * k, dtype=np.uint8)) for k in range(n)]
+    frames[0].save(path, save_all=True, append_images=frames[1:], duration=duration_ms, loop=0)
+
+
+def test_gif_frames() -> None:
+    with tempfile.TemporaryDirectory() as d:
+        path = os.path.join(d, "clip.gif")
+        make_gif(path)
+        clip = sample_video_frames(path, num_frames=16, fps=10.0, size=32, fps_nearest=True)
+        assert clip.shape == (16, 3, 32, 32), clip.shape
+        assert float(clip.max()) > 0, "GIF decoded as a black clip"
+        levels = (clip[:, 0, 0, 0] * 255).round().tolist()
+        assert levels == [10.0 * k for k in range(16)], levels      # source fps = target fps: frames 0..15 in order
+
+
+if __name__ == "__main__":
+    test_gif_frames()
+    print("GIF decode test passed")

trainer.py

@@ -18,6 +18,7 @@ from models.image.effort.effort import SynthScopeEffortModel
 from models.image.bfree.bfree import SynthScopeBFreeModel
 from models.image.d3qe.d3qe import SynthScopeD3QEModel
 from models.video.d3.d3 import SynthScopeD3Model
+from models.video.split.split import SynthScopeSPLITModel
 from models.video.restrav.restrav import SynthScopeReStraVModel
 from models.video.stall.stall import SynthScopeSTALLModel
 from metrics import AIGIMetrics
@@ -105,6 +106,8 @@ class SynthScopeModelTrainer:
             return SynthScopeD3QEModel(config)
         if model_type == "d3":
             return SynthScopeD3Model(config)
+        if model_type == "split":
+            return SynthScopeSPLITModel(config)
         if model_type == "restrav":
             return SynthScopeReStraVModel(config)
         if model_type == "stall":