wiltodelta commited on
Commit
b415bb6
·
verified ·
1 Parent(s): 090a97c

Source-classify freeze (GitHub f2a531e)

Browse files
Files changed (2) hide show
  1. README.md +1 -0
  2. benchmark.py +923 -219
README.md CHANGED
@@ -52,6 +52,7 @@ remove-ai-watermarks classify-source image.jpg --json
52
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/infer.py image.jpg --json
53
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/infer.py --inspect-weights
54
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/benchmark.py
 
55
  ```
56
 
57
  Try it in your browser on the [Remove AI Watermarks Hugging Face Space](https://huggingface.co/spaces/wiltodelta/remove-ai-watermarks) or at [raiw.cc](https://raiw.cc). Set `RAIW_SOURCE_CLASSIFY_WEIGHTS` to `multi-provider-source-v6.npz` or its parent directory for air-gapped deployments.
 
52
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/infer.py image.jpg --json
53
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/infer.py --inspect-weights
54
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/benchmark.py
55
+ uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/benchmark.py --tier frontier
56
  ```
57
 
58
  Try it in your browser on the [Remove AI Watermarks Hugging Face Space](https://huggingface.co/spaces/wiltodelta/remove-ai-watermarks) or at [raiw.cc](https://raiw.cc). Set `RAIW_SOURCE_CLASSIFY_WEIGHTS` to `multi-provider-source-v6.npz` or its parent directory for air-gapped deployments.
benchmark.py CHANGED
@@ -4,49 +4,63 @@
4
  # dependencies = [
5
  # "remove-ai-watermarks[source-classify]>=0.62.1",
6
  # "huggingface-hub>=0.26.0",
 
 
7
  # "pyarrow>=17.0.0",
8
- # "torch>=2.2.0",
9
- # "transformers>=4.46.0",
 
 
 
10
  # ]
11
  # ///
12
- """Reproducible open benchmark: `wiltodelta/raiw-source-classify` vs top Hugging Face binary AI detectors.
13
 
14
- Streams pinned, un-gated Parquet row groups directly from Hugging Face Hub over
15
- HTTP byte-ranges (no private files, no manual dataset setup) and evaluates:
16
  1. Non-AI false-positive traps (camera photos, UI screenshots, science diagrams,
17
- scanned receipts/invoices).
18
- 2. Modern AI generator outputs across multiple provider families (OpenAI, Google,
19
- Black Forest Labs FLUX, ByteDance Seedream, Midjourney, Stable Diffusion 3,
20
- Ideogram, xAI Grok).
21
- 3. Model artifact size, runtime stack, single-image warm CPU latency, binary
22
- precision/recall/FPR, and exact generator attribution accuracy.
 
 
 
23
 
24
  Run directly with `uv`:
25
 
26
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/benchmark.py
27
- uv run benchmark.py --json-out results.json
28
  uv run benchmark.py --raiw-only
29
  """
30
 
31
  from __future__ import annotations
32
 
33
  import argparse
 
34
  import hashlib
35
  import io
 
36
  import json
37
  import logging
 
38
  import statistics
39
  import sys
40
  import time
41
  from concurrent.futures import ThreadPoolExecutor
42
- from dataclasses import dataclass
43
  from pathlib import Path
44
- from typing import TYPE_CHECKING, Any
45
 
 
46
  from PIL import Image, ImageOps
47
 
48
  if TYPE_CHECKING:
49
- from huggingface_hub import HfFileSystem
 
 
50
 
51
  _REPO_SRC = Path(__file__).resolve().parents[2] / "src"
52
  if (_REPO_SRC / "remove_ai_watermarks").is_dir() and str(_REPO_SRC) not in sys.path:
@@ -54,11 +68,18 @@ if (_REPO_SRC / "remove_ai_watermarks").is_dir() and str(_REPO_SRC) not in sys.p
54
 
55
  log = logging.getLogger(__name__)
56
 
 
 
57
  MIN_SIDE_PX = 256
58
  MAX_AREA_PX = 9_000_000
59
- LATENCY_SAMPLE_COUNT = 40
 
60
  EXIF_ORIENTATION_TAG = 274
61
  TRANSPOSED_ORIENTATIONS = frozenset({5, 6, 7, 8})
 
 
 
 
62
 
63
 
64
  @dataclass(frozen=True)
@@ -72,6 +93,13 @@ class SampleItem:
72
  width: int
73
  height: int
74
  raw_bytes: bytes
 
 
 
 
 
 
 
75
 
76
  @property
77
  def is_ai(self) -> bool:
@@ -88,6 +116,19 @@ class ParquetSliceSpec:
88
  parquet_path: str
89
  limit: int = 50
90
  expected_provider: str | None = None
 
 
 
 
 
 
 
 
 
 
 
 
 
91
 
92
 
93
  @dataclass(frozen=True)
@@ -98,19 +139,52 @@ class PreferenceSliceSpec:
98
  revision: str
99
  parquet_path: str
100
  per_provider_cap: int
 
101
  max_row_groups: int | None = None
 
102
 
103
 
104
  @dataclass(frozen=True)
105
  class OpenDetectorSpec:
106
- """Pinned Hugging Face binary AI-image detector checkpoint."""
107
 
108
  repo_id: str
109
  revision: str
110
- ai_labels: frozenset[str]
111
  weights_mb: float
112
  license_id: str
 
 
 
 
 
 
 
 
113
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
 
115
  NON_AI_SLICES: tuple[ParquetSliceSpec, ...] = (
116
  ParquetSliceSpec(
@@ -143,37 +217,125 @@ NON_AI_SLICES: tuple[ParquetSliceSpec, ...] = (
143
  ),
144
  )
145
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
  RAPIDATA_4O_SPEC = PreferenceSliceSpec(
147
  dataset_id="Rapidata/OpenAI-4o_t2i_human_preference",
148
  revision="ca6c084c0fa0021d66bbdd44d6e61fbeb8e1198a",
149
  parquet_path="data/train-00000-of-00022.parquet",
150
  per_provider_cap=35,
 
151
  )
152
 
153
- NANO_BANANA_SPEC = ParquetSliceSpec(
154
  category="ai_google",
155
  dataset_id="bitmind/nano-banana",
156
  revision="9ea8da32a5be03f4946e6cb10c2d2f8e90f0a0a4",
157
  parquet_path="data/train-00000-of-00031.parquet",
158
  limit=25,
159
  expected_provider="google",
 
160
  )
161
 
162
- SEEDREAM_SPEC = PreferenceSliceSpec(
163
  dataset_id="Rapidata/Seedream-3_t2i_human_preference",
164
  revision="0867c34e5583d63376e3289b938e6a2cb6f43751",
165
  parquet_path="data/train-00000-of-00101.parquet",
166
  per_provider_cap=25,
 
167
  max_row_groups=3,
 
168
  )
169
 
170
- OPEN_BINARY_DETECTORS: tuple[OpenDetectorSpec, ...] = (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  OpenDetectorSpec(
172
- repo_id="umm-maybe/AI-image-detector",
173
- revision="c7e223baf11bc40528af364ba7bdea030ef42f9e",
174
  ai_labels=frozenset({"artificial"}),
175
- weights_mb=331.5,
176
- license_id="cc-by-4.0",
177
  ),
178
  OpenDetectorSpec(
179
  repo_id="Organika/sdxl-detector",
@@ -183,44 +345,64 @@ OPEN_BINARY_DETECTORS: tuple[OpenDetectorSpec, ...] = (
183
  license_id="cc-by-nc-3.0",
184
  ),
185
  OpenDetectorSpec(
186
- repo_id="Ateeqq/ai-vs-human-image-detector",
187
- revision="60e82406916921b823616bee33397baab38af3f0",
188
- ai_labels=frozenset({"ai"}),
189
- weights_mb=354.4,
190
  license_id="apache-2.0",
191
  ),
192
  OpenDetectorSpec(
193
- repo_id="haywoodsloan/ai-image-detector-deploy",
194
- revision="4ae1822603cb1ec980c217cfde1ef1b8e0477a16",
195
  ai_labels=frozenset({"artificial"}),
196
- weights_mb=744.7,
197
- license_id="apache-2.0",
198
  ),
199
  OpenDetectorSpec(
200
- repo_id="dima806/ai_vs_human_generated_image_detection",
201
- revision="341594f2b003d185db789becf48a2286c177b926",
202
- ai_labels=frozenset({"ai-generated", "artificial", "fake"}),
203
- weights_mb=327.3,
204
  license_id="apache-2.0",
205
  ),
206
  )
207
 
208
- RAPIDATA_4O_PROVIDER_MAP: dict[str, str] = {
209
- "4o-26-3-25": "openai",
210
- "dalle-3": "openai",
211
- "imagen-3": "google",
212
- "nano-banana": "google",
213
- "flux-1-pro": "flux",
214
- "flux-1.1-pro": "flux",
215
- "midjourney-5.2": "midjourney",
216
- "stable-diffusion-3": "stable_diffusion",
217
- "ideogram": "ideogram",
218
- "aurora-20-1-25": "xai",
219
- "seedream-3": "bytedance",
220
- "Seedream-3": "bytedance",
221
  }
222
 
223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
224
  def _inspect_image_bytes(raw_bytes: bytes) -> tuple[str, int, int] | None:
225
  try:
226
  with Image.open(io.BytesIO(raw_bytes)) as im:
@@ -243,12 +425,42 @@ def _decode_rgb(raw_bytes: bytes) -> Image.Image:
243
  return im.convert("RGB") if im.mode != "RGB" else im.copy()
244
 
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  def _try_build_sample(
247
  cell: Any,
248
  local_seen: set[str],
249
  *,
250
  category: str,
251
  expected_provider: str | None,
 
252
  ) -> SampleItem | None:
253
  raw = cell["bytes"] if isinstance(cell, dict) else cell
254
  if not isinstance(raw, (bytes, bytearray)):
@@ -270,15 +482,16 @@ def _try_build_sample(
270
  width=w,
271
  height=h,
272
  raw_bytes=raw_bytes,
 
273
  )
274
 
275
 
276
- def _latency_indices(total: int) -> list[int]:
277
  """Deterministic stratified indices across the full sample list for CPU timing."""
278
- if total <= LATENCY_SAMPLE_COUNT:
279
  return list(range(total))
280
- step = max(1, total // LATENCY_SAMPLE_COUNT)
281
- return [i * step for i in range(LATENCY_SAMPLE_COUNT)]
282
 
283
 
284
  def _read_single_image_slice(fs: HfFileSystem, spec: ParquetSliceSpec) -> list[SampleItem]:
@@ -289,11 +502,9 @@ def _read_single_image_slice(fs: HfFileSystem, spec: ParquetSliceSpec) -> list[S
289
  uri = f"datasets/{spec.dataset_id}/{spec.parquet_path}"
290
  with fs.open(uri, "rb", revision=spec.revision) as handle:
291
  pf = pq.ParquetFile(handle)
292
- for rg_idx in range(pf.num_row_groups):
293
- if len(items) >= spec.limit:
294
- break
295
- col = pf.read_row_group(rg_idx, columns=["image"])["image"]
296
- for row_idx in range(col.length()):
297
  if len(items) >= spec.limit:
298
  break
299
  sample = _try_build_sample(
@@ -301,31 +512,86 @@ def _read_single_image_slice(fs: HfFileSystem, spec: ParquetSliceSpec) -> list[S
301
  local_seen,
302
  category=spec.category,
303
  expected_provider=spec.expected_provider,
 
304
  )
305
  if sample is not None:
306
  items.append(sample)
 
 
307
  log.info("Loaded %s images from %s (%s)", len(items), spec.dataset_id, spec.category)
308
  return items
309
 
310
 
311
- def _read_preference_slice(
312
- fs: HfFileSystem,
313
- spec: PreferenceSliceSpec,
314
- *,
315
- seedream_only: bool = False,
316
- ) -> list[SampleItem]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  import pyarrow.parquet as pq
318
 
319
  items: list[SampleItem] = []
320
  local_seen: set[str] = set()
321
  counts: dict[str, int] = {}
322
- target_providers = {"bytedance"} if seedream_only else set(RAPIDATA_4O_PROVIDER_MAP.values())
 
323
  uri = f"datasets/{spec.dataset_id}/{spec.parquet_path}"
324
 
325
- def _map_tag(tag: str) -> str | None:
326
- if seedream_only:
327
- return "bytedance" if "seedream" in tag.lower() else None
328
- return RAPIDATA_4O_PROVIDER_MAP.get(tag)
 
329
 
330
  with fs.open(uri, "rb", revision=spec.revision) as handle:
331
  pf = pq.ParquetFile(handle)
@@ -334,21 +600,16 @@ def _read_preference_slice(
334
  if all(counts.get(p, 0) >= spec.per_provider_cap for p in target_providers):
335
  break
336
  meta_rg = pf.read_row_group(rg_idx, columns=["model1", "model2"])
337
- m1_list = meta_rg["model1"].to_pylist()
338
- m2_list = meta_rg["model2"].to_pylist()
339
- need_col1 = any(
340
- (p := _map_tag(str(t))) is not None and counts.get(p, 0) < spec.per_provider_cap for t in m1_list
341
- )
342
- need_col2 = any(
343
- (p := _map_tag(str(t))) is not None and counts.get(p, 0) < spec.per_provider_cap for t in m2_list
344
- )
345
  img_cols = [c for c, flag in (("image1", need_col1), ("image2", need_col2)) if flag]
346
  if not img_cols:
347
  continue
348
  img_rg = pf.read_row_group(rg_idx, columns=img_cols)
349
- for row_idx, (t1, t2) in enumerate(zip(m1_list, m2_list, strict=True)):
350
- for tag, icol in ((str(t1), "image1"), (str(t2), "image2")):
351
- prov = _map_tag(tag)
352
  if prov is None or counts.get(prov, 0) >= spec.per_provider_cap or icol not in img_cols:
353
  continue
354
  sample = _try_build_sample(
@@ -356,6 +617,7 @@ def _read_preference_slice(
356
  local_seen,
357
  category=f"ai_{prov}",
358
  expected_provider=prov,
 
359
  )
360
  if sample is not None:
361
  counts[prov] = counts.get(prov, 0) + 1
@@ -374,18 +636,90 @@ def _dedup_ordered(batches: list[list[SampleItem]], seen_shas: set[str]) -> list
374
  return merged
375
 
376
 
377
- def load_all_samples(fs: HfFileSystem) -> tuple[list[SampleItem], list[SampleItem]]:
378
- """Stream all 7 pinned non-AI and modern AI Parquet shards concurrently in deterministic order."""
379
- with ThreadPoolExecutor(max_workers=4) as pool:
380
  non_ai_futures = [pool.submit(_read_single_image_slice, fs, spec) for spec in NON_AI_SLICES]
381
- fut_4o = pool.submit(_read_preference_slice, fs, RAPIDATA_4O_SPEC, seedream_only=False)
382
- fut_nb = pool.submit(_read_single_image_slice, fs, NANO_BANANA_SPEC)
383
- fut_sd = pool.submit(_read_preference_slice, fs, SEEDREAM_SPEC, seedream_only=True)
 
 
384
 
385
  seen_shas: set[str] = set()
386
- non_ai = _dedup_ordered([f.result() for f in non_ai_futures], seen_shas)
387
- ai_samples = _dedup_ordered([fut_4o.result(), fut_nb.result(), fut_sd.result()], seen_shas)
388
- return non_ai, ai_samples
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
389
 
390
 
391
  def evaluate_raiw(samples: list[SampleItem]) -> dict[str, Any]:
@@ -405,11 +739,12 @@ def evaluate_raiw(samples: list[SampleItem]) -> dict[str, Any]:
405
 
406
  _classify_one(samples[0].raw_bytes)
407
 
408
- lat_idx_set = set(_latency_indices(len(samples)))
 
409
  latencies_ms: list[float] = []
410
  preds_by_idx: dict[int, dict[str, Any]] = {}
411
 
412
- for idx in sorted(lat_idx_set):
413
  t0 = time.perf_counter()
414
  preds_by_idx[idx] = _classify_one(samples[idx].raw_bytes)
415
  latencies_ms.append((time.perf_counter() - t0) * 1000.0)
@@ -420,7 +755,6 @@ def evaluate_raiw(samples: list[SampleItem]) -> dict[str, Any]:
420
  for idx, pred in zip(untimed_indices, untimed_preds, strict=True):
421
  preds_by_idx[idx] = pred
422
 
423
- preds = [preds_by_idx[i] for i in range(len(samples))]
424
  return _summarize_predictions(
425
  model_id=sc.WEIGHTS_REPO,
426
  revision=sc.WEIGHTS_REVISION,
@@ -429,78 +763,388 @@ def evaluate_raiw(samples: list[SampleItem]) -> dict[str, Any]:
429
  license_id="apache-2.0",
430
  median_cpu_ms=round(statistics.median(latencies_ms), 1),
431
  samples=samples,
432
- preds=preds,
 
433
  )
434
 
435
 
436
- def evaluate_hf_binary_detector(spec: OpenDetectorSpec, samples: list[SampleItem]) -> dict[str, Any]:
 
 
 
 
437
  """Evaluate an open Hugging Face `transformers` binary AI-image detector."""
438
  import torch
439
- from transformers import AutoImageProcessor, AutoModelForImageClassification
440
 
441
  processor = AutoImageProcessor.from_pretrained(spec.repo_id, revision=spec.revision)
442
- model = AutoModelForImageClassification.from_pretrained(spec.repo_id, revision=spec.revision)
443
- model.train(False)
444
- model.to("cpu")
445
-
446
- id2label = {int(k): str(v).lower() for k, v in model.config.id2label.items()}
447
- ai_indices = [idx for idx, lbl in id2label.items() if lbl in spec.ai_labels]
448
- if not ai_indices:
449
- raise RuntimeError(f"Could not map AI label for {spec.repo_id}: {id2label}")
 
450
 
451
- rgb0 = _decode_rgb(samples[0].raw_bytes)
452
- with torch.inference_mode():
453
- _ = model(**processor(images=rgb0, return_tensors="pt"))
 
454
 
455
- lat_idxs = _latency_indices(len(samples))
456
- lat_idx_set = set(lat_idxs)
457
- latencies_ms: list[float] = []
458
- preds_by_idx: dict[int, dict[str, Any]] = {}
459
 
460
- for idx in lat_idxs:
461
- t0 = time.perf_counter()
462
- rgb = _decode_rgb(samples[idx].raw_bytes)
463
- inputs = processor(images=rgb, return_tensors="pt")
464
- with torch.inference_mode():
465
- logits = model(**inputs).logits
466
- latencies_ms.append((time.perf_counter() - t0) * 1000.0)
467
- probs_1 = torch.softmax(logits, dim=-1)[0]
468
- p_ai = float(sum(probs_1[ai_idx].item() for ai_idx in ai_indices))
469
- preds_by_idx[idx] = {"pred_ai": p_ai >= 0.5, "pred_provider": None}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
470
 
471
- if torch.backends.mps.is_available():
472
- device = torch.device("mps")
473
- elif torch.cuda.is_available():
474
- device = torch.device("cuda")
475
- else:
476
- device = torch.device("cpu")
477
- model.to(device)
478
 
479
- untimed_indices = [idx for idx in range(len(samples)) if idx not in lat_idx_set]
480
- batch_size = 16
481
- for start in range(0, len(untimed_indices), batch_size):
482
- chunk_idxs = untimed_indices[start : start + batch_size]
483
- rgbs = [_decode_rgb(samples[idx].raw_bytes) for idx in chunk_idxs]
484
- inputs = {k: v.to(device) for k, v in processor(images=rgbs, return_tensors="pt").items()}
485
- with torch.inference_mode():
486
- probs = torch.softmax(model(**inputs).logits, dim=-1).cpu()
487
- for row_idx, sample_idx in enumerate(chunk_idxs):
488
- p_ai = float(sum(probs[row_idx, ai_idx].item() for ai_idx in ai_indices))
489
- preds_by_idx[sample_idx] = {"pred_ai": p_ai >= 0.5, "pred_provider": None}
 
 
 
 
 
 
 
 
490
 
491
- preds = [preds_by_idx[i] for i in range(len(samples))]
492
- return _summarize_predictions(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
493
  model_id=spec.repo_id,
494
  revision=spec.revision,
495
- runtime="PyTorch + Transformers",
496
  weights_mb=spec.weights_mb,
497
  license_id=spec.license_id,
498
- median_cpu_ms=round(statistics.median(latencies_ms), 1),
499
- samples=samples,
500
- preds=preds,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
501
  )
502
 
503
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
504
  def _summarize_predictions(
505
  *,
506
  model_id: str,
@@ -511,53 +1155,34 @@ def _summarize_predictions(
511
  median_cpu_ms: float,
512
  samples: list[SampleItem],
513
  preds: list[dict[str, Any]],
 
514
  ) -> dict[str, Any]:
515
  non_ai_by_cat: dict[str, list[bool]] = {}
516
- per_prov: dict[str, dict[str, int]] = {}
517
- tp = fp = tn = 0
518
- exact_provider_hits = 0
519
- ai_total = 0
520
 
521
  for item, pred in zip(samples, preds, strict=True):
522
- flagged = bool(pred["pred_ai"])
523
- if item.is_ai:
524
- ai_total += 1
525
- prov = item.expected_provider or "unknown"
526
- bucket = per_prov.setdefault(prov, {"count": 0, "detected_ai": 0, "exact_provider": 0})
527
- bucket["count"] += 1
528
- if flagged:
529
- tp += 1
530
- bucket["detected_ai"] += 1
531
- if pred["pred_provider"] is not None and pred["pred_provider"] == item.expected_provider:
532
- exact_provider_hits += 1
533
- bucket["exact_provider"] += 1
534
  else:
535
- non_ai_by_cat.setdefault(item.category, []).append(flagged)
536
- if flagged:
537
- fp += 1
538
- else:
539
- tn += 1
540
-
541
- non_ai_total = fp + tn
542
- precision = (tp / (tp + fp) * 100.0) if (tp + fp) else 0.0
543
- recall = (tp / ai_total * 100.0) if ai_total else 0.0
544
- fpr = (fp / non_ai_total * 100.0) if non_ai_total else 0.0
545
- attr_acc = (exact_provider_hits / ai_total * 100.0) if ai_total else 0.0
546
- attr_when_classified = (exact_provider_hits / tp * 100.0) if tp else 0.0
547
 
 
 
 
548
  slice_fprs = {cat: round(sum(flags) / len(flags) * 100.0, 1) for cat, flags in non_ai_by_cat.items()}
549
- per_provider_summary = {
550
- prov: {
551
- "count": b["count"],
552
- "detected_ai": b["detected_ai"],
553
- "exact_provider": b["exact_provider"],
554
- "recall_pct": round(b["detected_ai"] / b["count"] * 100.0, 1) if b["count"] else 0.0,
555
- "attribution_pct": round(b["exact_provider"] / b["count"] * 100.0, 1) if b["count"] else 0.0,
556
- }
557
- for prov, b in sorted(per_prov.items())
558
- }
559
  return {
560
  "model_id": model_id,
 
561
  "revision": revision,
562
  "license": license_id,
563
  "weights_mb": weights_mb,
@@ -567,76 +1192,155 @@ def _summarize_predictions(
567
  "non_ai_fp": fp,
568
  "non_ai_fpr_pct": round(fpr, 2),
569
  "slice_fpr_pct": slice_fprs,
570
- "ai_total": ai_total,
571
- "ai_tp": tp,
572
- "ai_recall_pct": round(recall, 1),
573
- "precision_pct": round(precision, 1),
574
- "provider_attribution_pct": round(attr_acc, 1),
575
- "provider_attribution_when_classified_pct": round(attr_when_classified, 1),
576
- "per_provider": per_provider_summary,
 
 
577
  }
578
 
579
 
580
  def format_markdown_table(results: list[dict[str, Any]]) -> str:
581
- """Format head-to-head results as a GitHub-flavored Markdown table."""
 
 
 
 
582
  header = (
583
- "| Model | Size | Runtime | CPU ms/img | Non-AI FPR (all) | Photo FPR "
584
- "| UI Screenshot FPR | Diagram FPR | Receipt FPR | AI Recall | Precision | Generator Attribution |"
 
585
  )
586
- sep = "|---|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|"
587
  lines = [header, sep]
588
  for r in results:
589
  sf = r["slice_fpr_pct"]
 
 
 
 
 
 
590
  lines.append(
591
- f"| `{r['model_id']}` | {r['weights_mb']:.1f} MB | {r['runtime']} | {r['median_cpu_ms']:.1f} ms "
592
  f"| {r['non_ai_fpr_pct']:.1f}% ({r['non_ai_fp']}/{r['non_ai_total']}) "
593
- f"| {sf.get('camera_photo', 0.0):.1f}% "
594
- f"| {sf.get('ui_screenshot', 0.0):.1f}% "
595
- f"| {sf.get('science_diagram', 0.0):.1f}% "
596
- f"| {sf.get('receipt_invoice', 0.0):.1f}% "
597
- f"| {r['ai_recall_pct']:.1f}% "
598
- f"| {r['precision_pct']:.1f}% "
599
- f"| {r['provider_attribution_pct']:.1f}% |"
600
  )
601
  return "\n".join(lines)
602
 
603
 
 
 
 
 
 
 
 
 
 
 
604
  def main() -> int:
605
- from huggingface_hub import HfFileSystem
606
 
607
  parser = argparse.ArgumentParser(description=__doc__)
608
  parser.add_argument("--json-out", type=Path, help="Optional path to write full JSON benchmark report.")
609
  parser.add_argument("--raiw-only", action="store_true", help="Run only wiltodelta/raiw-source-classify.")
 
 
 
 
 
 
610
  args = parser.parse_args()
611
 
612
  logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
613
  fs = HfFileSystem()
614
-
615
- non_ai, ai_samples = load_all_samples(fs)
616
- all_samples = non_ai + ai_samples
617
- log.info("Evaluating %s total images (%s non-AI + %s AI)", len(all_samples), len(non_ai), len(ai_samples))
 
 
 
 
 
 
 
 
 
618
 
619
  results: list[dict[str, Any]] = [evaluate_raiw(all_samples)]
 
 
620
  if not args.raiw_only:
621
- for spec in OPEN_BINARY_DETECTORS:
622
- log.info("Evaluating %s ...", spec.repo_id)
623
- results.append(evaluate_hf_binary_detector(spec, all_samples))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
624
 
625
  md_table = format_markdown_table(results)
626
  sys.stdout.write("\n" + md_table + "\n")
627
 
628
  if args.json_out is not None:
629
- format_counts: dict[str, dict[str, int]] = {}
630
- for x in ai_samples:
631
- prov = x.expected_provider or "unknown"
632
- fmap = format_counts.setdefault(prov, {})
633
- key = f"{x.container_format}:{x.width}x{x.height}"
634
- fmap[key] = fmap.get(key, 0) + 1
635
  payload = {
636
  "non_ai_count": len(non_ai),
637
- "ai_count": len(ai_samples),
638
- "ai_providers": {prov: sum(fmap.values()) for prov, fmap in sorted(format_counts.items())},
639
- "ai_provider_formats": format_counts,
 
 
 
640
  "results": results,
641
  }
642
  args.json_out.parent.mkdir(parents=True, exist_ok=True)
 
4
  # dependencies = [
5
  # "remove-ai-watermarks[source-classify]>=0.62.1",
6
  # "huggingface-hub>=0.26.0",
7
+ # "onnxruntime>=1.19.0",
8
+ # "peft>=0.14.0",
9
  # "pyarrow>=17.0.0",
10
+ # "safetensors>=0.4.5",
11
+ # "sentence-transformers>=6.1.0",
12
+ # "timm>=1.0.9",
13
+ # "torch>=2.5.0",
14
+ # "transformers>=5.20.0",
15
  # ]
16
  # ///
17
+ """Reproducible open benchmark: `wiltodelta/raiw-source-classify` vs 2025-2026 frontier and legacy HF AI detectors.
18
 
19
+ Streams pinned, un-gated datasets directly from Hugging Face Hub at immutable commit SHAs
20
+ (no private files, no manual dataset setup) across three evaluation regimes:
21
  1. Non-AI false-positive traps (camera photos, UI screenshots, science diagrams,
22
+ scanned receipts and invoices).
23
+ 2. Untouched native AI generator exports across FLUX.1-dev, OpenAI GPT-Image-1,
24
+ Google Gemini Nano Banana Pro, Stable Diffusion XL, and Midjourney v6.1.
25
+ 3. Transcoded crowd-arena AI outputs across multi-provider preference shards re-encoded
26
+ into square JPEG/WebP/PNG arena containers.
27
+
28
+ Evaluates open-weights configurations across artifact size, license, runtime stack,
29
+ warm single-image CPU latency, non-AI FPR by slice, binary recall/precision, and exact
30
+ generator-family attribution accuracy.
31
 
32
  Run directly with `uv`:
33
 
34
  uv run https://huggingface.co/wiltodelta/raiw-source-classify/resolve/main/benchmark.py
35
+ uv run benchmark.py --tier frontier --json-out results.json
36
  uv run benchmark.py --raiw-only
37
  """
38
 
39
  from __future__ import annotations
40
 
41
  import argparse
42
+ import gc
43
  import hashlib
44
  import io
45
+ import itertools
46
  import json
47
  import logging
48
+ import math
49
  import statistics
50
  import sys
51
  import time
52
  from concurrent.futures import ThreadPoolExecutor
53
+ from dataclasses import dataclass, field
54
  from pathlib import Path
55
+ from typing import TYPE_CHECKING, Any, Literal
56
 
57
+ import numpy as np
58
  from PIL import Image, ImageOps
59
 
60
  if TYPE_CHECKING:
61
+ from collections.abc import Callable
62
+
63
+ from huggingface_hub import HfApi, HfFileSystem
64
 
65
  _REPO_SRC = Path(__file__).resolve().parents[2] / "src"
66
  if (_REPO_SRC / "remove_ai_watermarks").is_dir() and str(_REPO_SRC) not in sys.path:
 
68
 
69
  log = logging.getLogger(__name__)
70
 
71
+ Regime = Literal["non_ai", "native", "arena"]
72
+
73
  MIN_SIDE_PX = 256
74
  MAX_AREA_PX = 9_000_000
75
+ LATENCY_SAMPLE_COUNT = 30
76
+ SMAUG_CPU_LATENCY_COUNT = 10
77
  EXIF_ORIENTATION_TAG = 274
78
  TRANSPOSED_ORIENTATIONS = frozenset({5, 6, 7, 8})
79
+ IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".webp")
80
+
81
+ IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32).reshape(3, 1, 1)
82
+ IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32).reshape(3, 1, 1)
83
 
84
 
85
  @dataclass(frozen=True)
 
93
  width: int
94
  height: int
95
  raw_bytes: bytes
96
+ regime: Regime = "arena"
97
+
98
+ def __post_init__(self) -> None:
99
+ if (self.regime == "non_ai") != (self.expected_provider is None):
100
+ raise ValueError(
101
+ f"Inconsistent SampleItem regime={self.regime!r} for expected_provider={self.expected_provider!r}"
102
+ )
103
 
104
  @property
105
  def is_ai(self) -> bool:
 
116
  parquet_path: str
117
  limit: int = 50
118
  expected_provider: str | None = None
119
+ regime: Regime = "non_ai"
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class RawRepoSliceSpec:
124
+ """Pinned Hugging Face dataset tree slice containing native generator export files."""
125
+
126
+ category: str
127
+ dataset_id: str
128
+ revision: str
129
+ path_in_repo: str | None
130
+ limit: int
131
+ expected_provider: str
132
 
133
 
134
  @dataclass(frozen=True)
 
139
  revision: str
140
  parquet_path: str
141
  per_provider_cap: int
142
+ provider_map: dict[str, str]
143
  max_row_groups: int | None = None
144
+ substring_match: bool = False
145
 
146
 
147
  @dataclass(frozen=True)
148
  class OpenDetectorSpec:
149
+ """Pinned Hugging Face `transformers` binary AI-image detector checkpoint."""
150
 
151
  repo_id: str
152
  revision: str
 
153
  weights_mb: float
154
  license_id: str
155
+ ai_labels: frozenset[str] = field(default_factory=frozenset)
156
+ tier: str = "legacy"
157
+ use_sigmoid: bool = False
158
+
159
+
160
+ @dataclass(frozen=True)
161
+ class TimmViTSpec:
162
+ """Pinned `vit_small_patch16_384` checkpoint evaluated at native CommFor preprocessing."""
163
 
164
+ repo_id: str
165
+ revision: str
166
+ weights_mb: float
167
+ license_id: str
168
+ logit_threshold: float
169
+
170
+
171
+ RAPIDATA_4O_PROVIDER_MAP: dict[str, str] = {
172
+ "4o-26-3-25": "openai",
173
+ "dalle-3": "openai",
174
+ "imagen-3": "google",
175
+ "nano-banana": "google",
176
+ "flux-1-pro": "flux",
177
+ "flux-1.1-pro": "flux",
178
+ "midjourney-5.2": "midjourney",
179
+ "stable-diffusion-3": "stable_diffusion",
180
+ "ideogram": "ideogram",
181
+ "aurora-20-1-25": "xai",
182
+ "seedream-3": "bytedance",
183
+ }
184
+
185
+ SEEDREAM_PROVIDER_MAP: dict[str, str] = {
186
+ "seedream": "bytedance",
187
+ }
188
 
189
  NON_AI_SLICES: tuple[ParquetSliceSpec, ...] = (
190
  ParquetSliceSpec(
 
217
  ),
218
  )
219
 
220
+ NATIVE_RAW_AI_SLICES: tuple[RawRepoSliceSpec, ...] = (
221
+ RawRepoSliceSpec(
222
+ category="native_flux",
223
+ dataset_id="LukasT9/Flux-1-Dev-Images-1k",
224
+ revision="ade8147d5f10bac016d691fd2b564e877e6315f7",
225
+ path_in_repo="images",
226
+ limit=25,
227
+ expected_provider="flux",
228
+ ),
229
+ RawRepoSliceSpec(
230
+ category="native_openai",
231
+ dataset_id="Tungtom2004/Output_dataset_gptImage1",
232
+ revision="ef327a0e5f089b34e27f25d2a7700c5fba811bf7",
233
+ path_in_repo="10000-20000",
234
+ limit=25,
235
+ expected_provider="openai",
236
+ ),
237
+ RawRepoSliceSpec(
238
+ category="native_stable_diffusion",
239
+ dataset_id="aziksh/animals_with_objects_sdxl",
240
+ revision="478f6a267926671d01d468e9211b5f5132d90aa3",
241
+ path_in_repo="cat_blue boat",
242
+ limit=25,
243
+ expected_provider="stable_diffusion",
244
+ ),
245
+ RawRepoSliceSpec(
246
+ category="native_midjourney",
247
+ dataset_id="saq1b/midjourney-v6.1",
248
+ revision="97a3ff2aea504384145af73d77df7f124613e9c6",
249
+ path_in_repo=None,
250
+ limit=25,
251
+ expected_provider="midjourney",
252
+ ),
253
+ )
254
+
255
+ NATIVE_GOOGLE_PARQUET_SPEC = ParquetSliceSpec(
256
+ category="native_google",
257
+ dataset_id="kaupane/nano-banana-pro-gen",
258
+ revision="5c61390a0307e6c38788ab85a5280fbc9893415b",
259
+ parquet_path="data/train-00000-of-00005.parquet",
260
+ limit=25,
261
+ expected_provider="google",
262
+ regime="native",
263
+ )
264
+
265
  RAPIDATA_4O_SPEC = PreferenceSliceSpec(
266
  dataset_id="Rapidata/OpenAI-4o_t2i_human_preference",
267
  revision="ca6c084c0fa0021d66bbdd44d6e61fbeb8e1198a",
268
  parquet_path="data/train-00000-of-00022.parquet",
269
  per_provider_cap=35,
270
+ provider_map=RAPIDATA_4O_PROVIDER_MAP,
271
  )
272
 
273
+ NANO_BANANA_ARENA_SPEC = ParquetSliceSpec(
274
  category="ai_google",
275
  dataset_id="bitmind/nano-banana",
276
  revision="9ea8da32a5be03f4946e6cb10c2d2f8e90f0a0a4",
277
  parquet_path="data/train-00000-of-00031.parquet",
278
  limit=25,
279
  expected_provider="google",
280
+ regime="arena",
281
  )
282
 
283
+ SEEDREAM_ARENA_SPEC = PreferenceSliceSpec(
284
  dataset_id="Rapidata/Seedream-3_t2i_human_preference",
285
  revision="0867c34e5583d63376e3289b938e6a2cb6f43751",
286
  parquet_path="data/train-00000-of-00101.parquet",
287
  per_provider_cap=25,
288
+ provider_map=SEEDREAM_PROVIDER_MAP,
289
  max_row_groups=3,
290
+ substring_match=True,
291
  )
292
 
293
+ FRONTIER_VIT_DETECTORS: tuple[TimmViTSpec, ...] = (
294
+ TimmViTSpec(
295
+ repo_id="TensorFold/detectra-v1",
296
+ revision="2bbc31161daa46932e05ee6f96e7016ee817bbde",
297
+ weights_mb=41.7,
298
+ license_id="mit",
299
+ logit_threshold=0.6190392, # logit(0.65)
300
+ ),
301
+ TimmViTSpec(
302
+ repo_id="Thermostatic/community-forensics-frontier-detector-2026-08",
303
+ revision="16db135220b318d811b207db576d90368980b595",
304
+ weights_mb=41.8,
305
+ license_id="mit",
306
+ logit_threshold=1.359375,
307
+ ),
308
+ )
309
+
310
+ COMMFOR_DEEPFAKEDET_SPEC = OpenDetectorSpec(
311
+ repo_id="buildborderless/CommunityForensics-DeepfakeDet-ViT",
312
+ revision="ac6ee457bea904a373065754107451793b56db00",
313
+ weights_mb=83.2,
314
+ license_id="mit",
315
+ tier="frontier",
316
+ use_sigmoid=True,
317
+ )
318
+
319
+ SMAUG_REPO = "abacusai/Smaug-AI-Image-Detector"
320
+ SMAUG_REVISION = "c7c95951e04a87b884a4ad479401b1128c3903bd"
321
+
322
+ DETECTRA_OPENFAKE_REPO = "s-gehring/detectra-openfake"
323
+ DETECTRA_OPENFAKE_REVISION = "c84565e5bf8969d95908e12844f551e77ae87105"
324
+
325
+ OPENFAKE_BASE_REPO = "bendavidsteel/openfake-swinv2-base"
326
+ OPENFAKE_BASE_REVISION = "90837e19c45b9a49af435d35071114ba1448c239"
327
+ OPENFAKE_ATTR_REPO = "bendavidsteel/openfake-swinv2-base-attribution-head"
328
+ OPENFAKE_ATTR_REVISION = "000944122ba497638fc88eea671dbe998fe635da"
329
+ OPENFAKE_LORA_REPO = "bendavidsteel/openfake-swinv2-base-textgraphics-lora"
330
+ OPENFAKE_LORA_REVISION = "b319ab4689286a620c3ef4a678709f4d04f25793"
331
+
332
+ LEGACY_BINARY_DETECTORS: tuple[OpenDetectorSpec, ...] = (
333
  OpenDetectorSpec(
334
+ repo_id="haywoodsloan/ai-image-detector-deploy",
335
+ revision="4ae1822603cb1ec980c217cfde1ef1b8e0477a16",
336
  ai_labels=frozenset({"artificial"}),
337
+ weights_mb=744.7,
338
+ license_id="apache-2.0",
339
  ),
340
  OpenDetectorSpec(
341
  repo_id="Organika/sdxl-detector",
 
345
  license_id="cc-by-nc-3.0",
346
  ),
347
  OpenDetectorSpec(
348
+ repo_id="dima806/ai_vs_human_generated_image_detection",
349
+ revision="341594f2b003d185db789becf48a2286c177b926",
350
+ ai_labels=frozenset({"ai-generated", "artificial", "fake"}),
351
+ weights_mb=327.3,
352
  license_id="apache-2.0",
353
  ),
354
  OpenDetectorSpec(
355
+ repo_id="umm-maybe/AI-image-detector",
356
+ revision="c7e223baf11bc40528af364ba7bdea030ef42f9e",
357
  ai_labels=frozenset({"artificial"}),
358
+ weights_mb=331.5,
359
+ license_id="cc-by-4.0",
360
  ),
361
  OpenDetectorSpec(
362
+ repo_id="Ateeqq/ai-vs-human-image-detector",
363
+ revision="60e82406916921b823616bee33397baab38af3f0",
364
+ ai_labels=frozenset({"ai"}),
365
+ weights_mb=354.4,
366
  license_id="apache-2.0",
367
  ),
368
  )
369
 
370
+ OPENFAKE_FAMILY_MAP: dict[str, str] = {
371
+ "DALLE3": "openai",
372
+ "GPT-Image": "openai",
373
+ "Imagen": "google",
374
+ "FLUX1": "flux",
375
+ "Midjourney": "midjourney",
376
+ "SD1": "stable_diffusion",
377
+ "SD2": "stable_diffusion",
378
+ "SD3": "stable_diffusion",
379
+ "SDXL": "stable_diffusion",
380
+ "Ideogram": "ideogram",
381
+ "Grok": "xai",
 
382
  }
383
 
384
 
385
+ def _sigmoid(x: float) -> float:
386
+ if x >= 0.0:
387
+ z = math.exp(-x)
388
+ return 1.0 / (1.0 + z)
389
+ z = math.exp(x)
390
+ return z / (1.0 + z)
391
+
392
+
393
+ def _release_memory() -> None:
394
+ gc.collect()
395
+ try:
396
+ import torch
397
+
398
+ if torch.cuda.is_available():
399
+ torch.cuda.empty_cache()
400
+ if torch.backends.mps.is_available():
401
+ torch.mps.empty_cache()
402
+ except ImportError:
403
+ return
404
+
405
+
406
  def _inspect_image_bytes(raw_bytes: bytes) -> tuple[str, int, int] | None:
407
  try:
408
  with Image.open(io.BytesIO(raw_bytes)) as im:
 
425
  return im.convert("RGB") if im.mode != "RGB" else im.copy()
426
 
427
 
428
+ def _preprocess_commfor_chw(rgb: Image.Image, short_edge: int = 440, crop: int = 384) -> np.ndarray:
429
+ w, h = rgb.size
430
+ scale = short_edge / min(w, h)
431
+ nw, nh = round(w * scale), round(h * scale)
432
+ resized = rgb.resize((nw, nh), Image.Resampling.BILINEAR)
433
+ left = (nw - crop) // 2
434
+ top = (nh - crop) // 2
435
+ cropped = resized.crop((left, top, left + crop, top + crop))
436
+ arr = np.asarray(cropped, dtype=np.float32) / 255.0
437
+ chw = np.transpose(arr, (2, 0, 1))
438
+ return (chw - IMAGENET_MEAN) / IMAGENET_STD
439
+
440
+
441
+ def _smaug_views(rgb: Image.Image) -> list[Image.Image]:
442
+ crop_px, side_px = 528, 576
443
+ w, h = rgb.size
444
+ if min(w, h) > side_px:
445
+ scale = side_px / min(w, h)
446
+ rgb = rgb.resize((round(w * scale), round(h * scale)), Image.Resampling.LANCZOS)
447
+ elif min(w, h) < crop_px:
448
+ scale = crop_px / min(w, h)
449
+ rgb = rgb.resize((math.ceil(w * scale), math.ceil(h * scale)), Image.Resampling.BICUBIC)
450
+ w, h = rgb.size
451
+ cx, cy = (w - crop_px) // 2, (h - crop_px) // 2
452
+ if w >= h:
453
+ return [rgb.crop((x, cy, x + crop_px, cy + crop_px)) for x in (0, cx, w - crop_px)]
454
+ return [rgb.crop((cx, y, cx + crop_px, y + crop_px)) for y in (0, cy, h - crop_px)]
455
+
456
+
457
  def _try_build_sample(
458
  cell: Any,
459
  local_seen: set[str],
460
  *,
461
  category: str,
462
  expected_provider: str | None,
463
+ regime: Regime,
464
  ) -> SampleItem | None:
465
  raw = cell["bytes"] if isinstance(cell, dict) else cell
466
  if not isinstance(raw, (bytes, bytearray)):
 
482
  width=w,
483
  height=h,
484
  raw_bytes=raw_bytes,
485
+ regime=regime,
486
  )
487
 
488
 
489
+ def _latency_indices(total: int, count: int = LATENCY_SAMPLE_COUNT) -> list[int]:
490
  """Deterministic stratified indices across the full sample list for CPU timing."""
491
+ if total <= count:
492
  return list(range(total))
493
+ step = max(1, total // count)
494
+ return [i * step for i in range(count)]
495
 
496
 
497
  def _read_single_image_slice(fs: HfFileSystem, spec: ParquetSliceSpec) -> list[SampleItem]:
 
502
  uri = f"datasets/{spec.dataset_id}/{spec.parquet_path}"
503
  with fs.open(uri, "rb", revision=spec.revision) as handle:
504
  pf = pq.ParquetFile(handle)
505
+ for batch in pf.iter_batches(batch_size=spec.limit + 16, columns=["image"]):
506
+ col = batch["image"]
507
+ for row_idx in range(len(col)):
 
 
508
  if len(items) >= spec.limit:
509
  break
510
  sample = _try_build_sample(
 
512
  local_seen,
513
  category=spec.category,
514
  expected_provider=spec.expected_provider,
515
+ regime=spec.regime,
516
  )
517
  if sample is not None:
518
  items.append(sample)
519
+ if len(items) >= spec.limit:
520
+ break
521
  log.info("Loaded %s images from %s (%s)", len(items), spec.dataset_id, spec.category)
522
  return items
523
 
524
 
525
+ def _read_raw_repo_slice(api: HfApi, spec: RawRepoSliceSpec) -> list[SampleItem]:
526
+ from huggingface_hub import hf_hub_download
527
+
528
+ tree_iter = api.list_repo_tree(
529
+ spec.dataset_id,
530
+ path_in_repo=spec.path_in_repo,
531
+ repo_type="dataset",
532
+ revision=spec.revision,
533
+ )
534
+ candidate_paths = list(
535
+ itertools.islice(
536
+ (
537
+ str(entry.path)
538
+ for entry in tree_iter
539
+ if str(getattr(entry, "path", "")).lower().endswith(IMAGE_SUFFIXES)
540
+ ),
541
+ spec.limit + 8,
542
+ )
543
+ )
544
+
545
+ def _download_bytes(rel_path: str) -> bytes:
546
+ local_path = Path(
547
+ hf_hub_download(
548
+ spec.dataset_id,
549
+ rel_path,
550
+ repo_type="dataset",
551
+ revision=spec.revision,
552
+ )
553
+ )
554
+ return local_path.read_bytes()
555
+
556
+ items: list[SampleItem] = []
557
+ local_seen: set[str] = set()
558
+ offset = 0
559
+ with ThreadPoolExecutor(max_workers=8) as pool:
560
+ while len(items) < spec.limit and offset < len(candidate_paths):
561
+ needed = spec.limit - len(items)
562
+ batch_paths = candidate_paths[offset : offset + needed]
563
+ offset += len(batch_paths)
564
+ for blob in pool.map(_download_bytes, batch_paths):
565
+ sample = _try_build_sample(
566
+ blob,
567
+ local_seen,
568
+ category=spec.category,
569
+ expected_provider=spec.expected_provider,
570
+ regime="native",
571
+ )
572
+ if sample is not None:
573
+ items.append(sample)
574
+ if len(items) >= spec.limit:
575
+ break
576
+ log.info("Loaded %s native images from %s (%s)", len(items), spec.dataset_id, spec.category)
577
+ return items
578
+
579
+
580
+ def _read_preference_slice(fs: HfFileSystem, spec: PreferenceSliceSpec) -> list[SampleItem]:
581
  import pyarrow.parquet as pq
582
 
583
  items: list[SampleItem] = []
584
  local_seen: set[str] = set()
585
  counts: dict[str, int] = {}
586
+ norm_map = {k.lower(): v for k, v in spec.provider_map.items()}
587
+ target_providers = set(norm_map.values())
588
  uri = f"datasets/{spec.dataset_id}/{spec.parquet_path}"
589
 
590
+ def _resolve_provider(raw_tag: Any) -> str | None:
591
+ tag_lc = str(raw_tag).lower()
592
+ if spec.substring_match:
593
+ return next((prov for needle, prov in norm_map.items() if needle in tag_lc), None)
594
+ return norm_map.get(tag_lc)
595
 
596
  with fs.open(uri, "rb", revision=spec.revision) as handle:
597
  pf = pq.ParquetFile(handle)
 
600
  if all(counts.get(p, 0) >= spec.per_provider_cap for p in target_providers):
601
  break
602
  meta_rg = pf.read_row_group(rg_idx, columns=["model1", "model2"])
603
+ p1_list = [_resolve_provider(t) for t in meta_rg["model1"].to_pylist()]
604
+ p2_list = [_resolve_provider(t) for t in meta_rg["model2"].to_pylist()]
605
+ need_col1 = any(p is not None and counts.get(p, 0) < spec.per_provider_cap for p in p1_list)
606
+ need_col2 = any(p is not None and counts.get(p, 0) < spec.per_provider_cap for p in p2_list)
 
 
 
 
607
  img_cols = [c for c, flag in (("image1", need_col1), ("image2", need_col2)) if flag]
608
  if not img_cols:
609
  continue
610
  img_rg = pf.read_row_group(rg_idx, columns=img_cols)
611
+ for row_idx, (p1, p2) in enumerate(zip(p1_list, p2_list, strict=True)):
612
+ for prov, icol in ((p1, "image1"), (p2, "image2")):
 
613
  if prov is None or counts.get(prov, 0) >= spec.per_provider_cap or icol not in img_cols:
614
  continue
615
  sample = _try_build_sample(
 
617
  local_seen,
618
  category=f"ai_{prov}",
619
  expected_provider=prov,
620
+ regime="arena",
621
  )
622
  if sample is not None:
623
  counts[prov] = counts.get(prov, 0) + 1
 
636
  return merged
637
 
638
 
639
+ def load_all_samples(fs: HfFileSystem, api: HfApi) -> list[SampleItem]:
640
+ """Stream all pinned non-AI, native-export AI, and crowd-arena AI slices in deterministic order."""
641
+ with ThreadPoolExecutor(max_workers=6) as pool:
642
  non_ai_futures = [pool.submit(_read_single_image_slice, fs, spec) for spec in NON_AI_SLICES]
643
+ native_raw_futures = [pool.submit(_read_raw_repo_slice, api, spec) for spec in NATIVE_RAW_AI_SLICES]
644
+ fut_native_google = pool.submit(_read_single_image_slice, fs, NATIVE_GOOGLE_PARQUET_SPEC)
645
+ fut_4o = pool.submit(_read_preference_slice, fs, RAPIDATA_4O_SPEC)
646
+ fut_nb = pool.submit(_read_single_image_slice, fs, NANO_BANANA_ARENA_SPEC)
647
+ fut_sd = pool.submit(_read_preference_slice, fs, SEEDREAM_ARENA_SPEC)
648
 
649
  seen_shas: set[str] = set()
650
+ return _dedup_ordered(
651
+ [
652
+ *[f.result() for f in non_ai_futures],
653
+ *[f.result() for f in native_raw_futures],
654
+ fut_native_google.result(),
655
+ fut_4o.result(),
656
+ fut_nb.result(),
657
+ fut_sd.result(),
658
+ ],
659
+ seen_shas,
660
+ )
661
+
662
+
663
+ def _select_torch_accelerator() -> Any:
664
+ import torch
665
+
666
+ if torch.cuda.is_available():
667
+ return torch.device("cuda")
668
+ if torch.backends.mps.is_available():
669
+ return torch.device("mps")
670
+ return torch.device("cpu")
671
+
672
+
673
+ def _evaluate_rgb_detector(
674
+ samples: list[SampleItem],
675
+ rgb_cache: list[Image.Image] | None,
676
+ *,
677
+ score_cpu_batch: Callable[[list[Image.Image]], list[dict[str, Any]]],
678
+ make_batch_scorer: Callable[[], Callable[[list[Image.Image]], list[dict[str, Any]]]] | None = None,
679
+ batch_size: int = 16,
680
+ latency_count: int = LATENCY_SAMPLE_COUNT,
681
+ model_id: str,
682
+ revision: str,
683
+ runtime: str,
684
+ weights_mb: float,
685
+ license_id: str,
686
+ tier: str = "frontier",
687
+ ) -> dict[str, Any]:
688
+ """Shared harness: warm up on CPU, measure single-image CPU latency from raw bytes, then batch untimed items."""
689
+ rgb0 = rgb_cache[0] if rgb_cache is not None else _decode_rgb(samples[0].raw_bytes)
690
+ _ = score_cpu_batch([rgb0])
691
+
692
+ lat_idxs = _latency_indices(len(samples), latency_count)
693
+ lat_idx_set = set(lat_idxs)
694
+ latencies_ms: list[float] = []
695
+ preds_by_idx: dict[int, dict[str, Any]] = {}
696
+
697
+ for idx in lat_idxs:
698
+ t0 = time.perf_counter()
699
+ rgb = _decode_rgb(samples[idx].raw_bytes)
700
+ pred = score_cpu_batch([rgb])[0]
701
+ latencies_ms.append((time.perf_counter() - t0) * 1000.0)
702
+ preds_by_idx[idx] = pred
703
+
704
+ batch_scorer = make_batch_scorer() if make_batch_scorer is not None else score_cpu_batch
705
+ untimed_indices = [idx for idx in range(len(samples)) if idx not in lat_idx_set]
706
+ for start in range(0, len(untimed_indices), batch_size):
707
+ chunk_idxs = untimed_indices[start : start + batch_size]
708
+ rgbs = [rgb_cache[idx] if rgb_cache is not None else _decode_rgb(samples[idx].raw_bytes) for idx in chunk_idxs]
709
+ for sample_idx, pred in zip(chunk_idxs, batch_scorer(rgbs), strict=True):
710
+ preds_by_idx[sample_idx] = pred
711
+
712
+ return _summarize_predictions(
713
+ model_id=model_id,
714
+ revision=revision,
715
+ runtime=runtime,
716
+ weights_mb=weights_mb,
717
+ license_id=license_id,
718
+ median_cpu_ms=round(statistics.median(latencies_ms), 1),
719
+ samples=samples,
720
+ preds=[preds_by_idx[i] for i in range(len(samples))],
721
+ tier=tier,
722
+ )
723
 
724
 
725
  def evaluate_raiw(samples: list[SampleItem]) -> dict[str, Any]:
 
739
 
740
  _classify_one(samples[0].raw_bytes)
741
 
742
+ lat_idxs = _latency_indices(len(samples))
743
+ lat_idx_set = set(lat_idxs)
744
  latencies_ms: list[float] = []
745
  preds_by_idx: dict[int, dict[str, Any]] = {}
746
 
747
+ for idx in lat_idxs:
748
  t0 = time.perf_counter()
749
  preds_by_idx[idx] = _classify_one(samples[idx].raw_bytes)
750
  latencies_ms.append((time.perf_counter() - t0) * 1000.0)
 
755
  for idx, pred in zip(untimed_indices, untimed_preds, strict=True):
756
  preds_by_idx[idx] = pred
757
 
 
758
  return _summarize_predictions(
759
  model_id=sc.WEIGHTS_REPO,
760
  revision=sc.WEIGHTS_REVISION,
 
763
  license_id="apache-2.0",
764
  median_cpu_ms=round(statistics.median(latencies_ms), 1),
765
  samples=samples,
766
+ preds=[preds_by_idx[i] for i in range(len(samples))],
767
+ tier="raiw",
768
  )
769
 
770
 
771
+ def evaluate_hf_binary_detector(
772
+ spec: OpenDetectorSpec,
773
+ samples: list[SampleItem],
774
+ rgb_cache: list[Image.Image] | None = None,
775
+ ) -> dict[str, Any]:
776
  """Evaluate an open Hugging Face `transformers` binary AI-image detector."""
777
  import torch
778
+ from transformers import AutoImageProcessor, AutoModelForImageClassification, ViTForImageClassification
779
 
780
  processor = AutoImageProcessor.from_pretrained(spec.repo_id, revision=spec.revision)
781
+ ai_indices: list[int] = []
782
+ if spec.use_sigmoid:
783
+ model = ViTForImageClassification.from_pretrained(spec.repo_id, revision=spec.revision)
784
+ else:
785
+ model = AutoModelForImageClassification.from_pretrained(spec.repo_id, revision=spec.revision)
786
+ id2label = {int(k): str(v).lower() for k, v in model.config.id2label.items()}
787
+ ai_indices = [idx for idx, lbl in id2label.items() if lbl in spec.ai_labels]
788
+ if not ai_indices:
789
+ raise RuntimeError(f"Could not map AI label for {spec.repo_id}: {id2label}")
790
 
791
+ def logits_to_probs(logits: Any) -> Any:
792
+ if spec.use_sigmoid:
793
+ return torch.sigmoid(logits.reshape(-1))
794
+ return torch.softmax(logits, dim=-1)[:, ai_indices].sum(dim=-1)
795
 
796
+ model.train(False)
797
+ model.to("cpu")
 
 
798
 
799
+ def _score_on(dev: Any) -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
800
+ def _run(rgbs: list[Image.Image]) -> list[dict[str, Any]]:
801
+ inputs = {k: v.to(dev) for k, v in processor(images=rgbs, return_tensors="pt").items()}
802
+ with torch.inference_mode():
803
+ probs = logits_to_probs(model(**inputs).logits).cpu().tolist()
804
+ return [{"pred_ai": float(p_val) >= 0.5, "pred_provider": None} for p_val in probs]
805
+
806
+ return _run
807
+
808
+ def _move_to_acc() -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
809
+ dev = _select_torch_accelerator()
810
+ model.to(dev)
811
+ return _score_on(dev)
812
+
813
+ return _evaluate_rgb_detector(
814
+ samples,
815
+ rgb_cache,
816
+ score_cpu_batch=_score_on(torch.device("cpu")),
817
+ make_batch_scorer=_move_to_acc,
818
+ model_id=spec.repo_id,
819
+ revision=spec.revision,
820
+ runtime="PyTorch + Transformers",
821
+ weights_mb=spec.weights_mb,
822
+ license_id=spec.license_id,
823
+ tier=spec.tier,
824
+ )
825
 
 
 
 
 
 
 
 
826
 
827
+ def evaluate_timm_vit_detector(
828
+ spec: TimmViTSpec,
829
+ samples: list[SampleItem],
830
+ rgb_cache: list[Image.Image] | None = None,
831
+ ) -> dict[str, Any]:
832
+ """Evaluate a `vit_small_patch16_384` frontier detector at official 440->384 preprocessing."""
833
+ import timm
834
+ import torch
835
+ from huggingface_hub import hf_hub_download
836
+ from safetensors.torch import load_file
837
+
838
+ raw_state = load_file(hf_hub_download(spec.repo_id, "model.safetensors", revision=spec.revision))
839
+ clean_state = {(k[4:] if k.startswith("vit.") else k): v for k, v in raw_state.items()}
840
+ del raw_state
841
+ model = timm.create_model("vit_small_patch16_384", pretrained=False, num_classes=1)
842
+ model.load_state_dict(clean_state, strict=True)
843
+ del clean_state
844
+ model.train(False)
845
+ model.to("cpu")
846
 
847
+ def _score_on(dev: Any) -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
848
+ def _run(rgbs: list[Image.Image]) -> list[dict[str, Any]]:
849
+ batch_np = np.stack([_preprocess_commfor_chw(rgb) for rgb in rgbs])
850
+ with torch.inference_mode():
851
+ logits = model(torch.from_numpy(batch_np).to(dev)).reshape(-1).cpu().tolist()
852
+ return [{"pred_ai": float(val) >= spec.logit_threshold, "pred_provider": None} for val in logits]
853
+
854
+ return _run
855
+
856
+ def _move_to_acc() -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
857
+ dev = _select_torch_accelerator()
858
+ model.to(dev)
859
+ return _score_on(dev)
860
+
861
+ return _evaluate_rgb_detector(
862
+ samples,
863
+ rgb_cache,
864
+ score_cpu_batch=_score_on(torch.device("cpu")),
865
+ make_batch_scorer=_move_to_acc,
866
  model_id=spec.repo_id,
867
  revision=spec.revision,
868
+ runtime="PyTorch + timm / ONNX",
869
  weights_mb=spec.weights_mb,
870
  license_id=spec.license_id,
871
+ tier="frontier",
872
+ )
873
+
874
+
875
+ def evaluate_detectra_openfake(
876
+ samples: list[SampleItem],
877
+ rgb_cache: list[Image.Image] | None = None,
878
+ ) -> dict[str, Any]:
879
+ """Evaluate `s-gehring/detectra-openfake` INT8 ONNX with its documented two-pass Platt rule."""
880
+ import onnxruntime as ort
881
+ from huggingface_hub import hf_hub_download
882
+
883
+ onnx_path = Path(hf_hub_download(DETECTRA_OPENFAKE_REPO, "model.onnx", revision=DETECTRA_OPENFAKE_REVISION))
884
+ weights_mb = round(onnx_path.stat().st_size / (1024 * 1024), 2)
885
+ sess_opts = ort.SessionOptions()
886
+ sess_opts.intra_op_num_threads = 4
887
+ sess = ort.InferenceSession(str(onnx_path), sess_options=sess_opts, providers=["CPUExecutionProvider"])
888
+ in_name = sess.get_inputs()[0].name
889
+
890
+ def _platt(logit_val: float) -> float:
891
+ return _sigmoid(0.552 * logit_val + 0.1933)
892
+
893
+ def _score_batch(rgbs: list[Image.Image]) -> list[dict[str, Any]]:
894
+ x_batch = np.stack([_preprocess_commfor_chw(rgb, short_edge=440, crop=384) for rgb in rgbs])
895
+ logits = np.asarray(sess.run(None, {in_name: x_batch})[0]).reshape(-1)
896
+ probs = [_platt(float(val)) for val in logits]
897
+
898
+ borderline = [idx for idx, p_val in enumerate(probs) if 0.5 <= p_val < 0.95]
899
+ if borderline:
900
+ hi_crops = np.stack(
901
+ [_preprocess_commfor_chw(rgbs[idx], short_edge=se, crop=384) for idx in borderline for se in (576, 768)]
902
+ )
903
+ hi_logits = np.asarray(sess.run(None, {in_name: hi_crops})[0]).reshape(len(borderline), 2)
904
+ for b_pos, idx in enumerate(borderline):
905
+ probs[idx] = max(probs[idx], _platt(float(hi_logits[b_pos, 0])), _platt(float(hi_logits[b_pos, 1])))
906
+
907
+ return [{"pred_ai": p_val >= 0.95, "pred_provider": None} for p_val in probs]
908
+
909
+ return _evaluate_rgb_detector(
910
+ samples,
911
+ rgb_cache,
912
+ score_cpu_batch=_score_batch,
913
+ model_id=DETECTRA_OPENFAKE_REPO,
914
+ revision=DETECTRA_OPENFAKE_REVISION,
915
+ runtime="ONNX Runtime INT8 (CPU)",
916
+ weights_mb=weights_mb,
917
+ license_id="cc-by-nc-4.0",
918
+ tier="frontier",
919
+ )
920
+
921
+
922
+ def evaluate_openfake_attribution(
923
+ samples: list[SampleItem],
924
+ rgb_cache: list[Image.Image] | None = None,
925
+ ) -> dict[str, Any]:
926
+ """Evaluate OpenFake SwinV2-base paired with its 15-family sub-center attribution head."""
927
+ import torch
928
+ import torch.nn as nn
929
+ import torch.nn.functional as f_nn
930
+ from huggingface_hub import hf_hub_download
931
+ from safetensors.torch import load_file
932
+ from transformers import AutoImageProcessor, AutoModelForImageClassification
933
+
934
+ base_st = Path(hf_hub_download(OPENFAKE_BASE_REPO, "model.safetensors", revision=OPENFAKE_BASE_REVISION))
935
+ attr_cfg_path = Path(hf_hub_download(OPENFAKE_ATTR_REPO, "config.json", revision=OPENFAKE_ATTR_REVISION))
936
+ attr_st_path = Path(hf_hub_download(OPENFAKE_ATTR_REPO, "model.safetensors", revision=OPENFAKE_ATTR_REVISION))
937
+ attr_mb = round((base_st.stat().st_size + attr_st_path.stat().st_size) / (1024 * 1024), 2)
938
+
939
+ attr_cfg = json.loads(attr_cfg_path.read_text(encoding="utf-8"))
940
+ attr_weights = load_file(str(attr_st_path))
941
+ families: list[str] = [str(x) for x in attr_cfg["families"]]
942
+
943
+ processor = AutoImageProcessor.from_pretrained(OPENFAKE_BASE_REPO, revision=OPENFAKE_BASE_REVISION)
944
+ detector = AutoModelForImageClassification.from_pretrained(OPENFAKE_BASE_REPO, revision=OPENFAKE_BASE_REVISION)
945
+ detector.train(False)
946
+ detector.to("cpu")
947
+
948
+ head = nn.Sequential(
949
+ nn.Linear(int(attr_cfg["in_dim"]), int(attr_cfg["emb_dim"])),
950
+ nn.BatchNorm1d(int(attr_cfg["emb_dim"])),
951
+ )
952
+ head.load_state_dict({k[len("head.") :]: v for k, v in attr_weights.items() if k.startswith("head.")})
953
+ head.train(False)
954
+ head.to("cpu")
955
+ family_protos_cpu = attr_weights["family_prototypes"].float()
956
+ del attr_weights
957
+
958
+ def _score_on(dev: Any, protos: Any) -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
959
+ def _run(rgbs: list[Image.Image]) -> list[dict[str, Any]]:
960
+ px = processor(images=rgbs, return_tensors="pt")["pixel_values"].to(dev)
961
+ with torch.inference_mode():
962
+ pooled = detector.swinv2(pixel_values=px).pooler_output
963
+ p_fake = (detector.classifier(pooled) / 2.0).softmax(-1)[:, 1]
964
+ emb = f_nn.normalize(head(f_nn.normalize(pooled)))
965
+ fam_idx = (emb @ protos.T).argmax(dim=-1)
966
+ return [
967
+ {
968
+ "pred_ai": float(p_val) >= 0.5,
969
+ "pred_provider": (
970
+ OPENFAKE_FAMILY_MAP.get(families[int(f_i)], f"other:{families[int(f_i)]}")
971
+ if float(p_val) >= 0.5
972
+ else None
973
+ ),
974
+ }
975
+ for p_val, f_i in zip(p_fake.cpu().tolist(), fam_idx.cpu().tolist(), strict=True)
976
+ ]
977
+
978
+ return _run
979
+
980
+ def _move_to_acc() -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
981
+ dev = _select_torch_accelerator()
982
+ detector.to(dev)
983
+ head.to(dev)
984
+ return _score_on(dev, family_protos_cpu.to(dev))
985
+
986
+ return _evaluate_rgb_detector(
987
+ samples,
988
+ rgb_cache,
989
+ score_cpu_batch=_score_on(torch.device("cpu"), family_protos_cpu),
990
+ make_batch_scorer=_move_to_acc,
991
+ model_id="bendavidsteel/openfake-swinv2-base + attribution-head",
992
+ revision=f"{OPENFAKE_BASE_REVISION[:10]}+{OPENFAKE_ATTR_REVISION[:10]}",
993
+ runtime="PyTorch + Transformers",
994
+ weights_mb=attr_mb,
995
+ license_id="cc-by-nc-4.0",
996
+ tier="frontier",
997
+ )
998
+
999
+
1000
+ def evaluate_openfake_textgraphics_lora(
1001
+ samples: list[SampleItem],
1002
+ rgb_cache: list[Image.Image] | None = None,
1003
+ ) -> dict[str, Any]:
1004
+ """Evaluate OpenFake SwinV2-base with the McGill text-graphics false-positive LoRA adapter."""
1005
+ import torch
1006
+ from huggingface_hub import hf_hub_download
1007
+ from peft import PeftModel
1008
+ from transformers import AutoImageProcessor, AutoModelForImageClassification
1009
+
1010
+ base_st = Path(hf_hub_download(OPENFAKE_BASE_REPO, "model.safetensors", revision=OPENFAKE_BASE_REVISION))
1011
+ lora_st = Path(hf_hub_download(OPENFAKE_LORA_REPO, "adapter_model.safetensors", revision=OPENFAKE_LORA_REVISION))
1012
+ lora_mb = round((base_st.stat().st_size + lora_st.stat().st_size) / (1024 * 1024), 2)
1013
+
1014
+ processor = AutoImageProcessor.from_pretrained(OPENFAKE_BASE_REPO, revision=OPENFAKE_BASE_REVISION)
1015
+ base_model = AutoModelForImageClassification.from_pretrained(OPENFAKE_BASE_REPO, revision=OPENFAKE_BASE_REVISION)
1016
+ lora_model = PeftModel.from_pretrained(base_model, OPENFAKE_LORA_REPO, revision=OPENFAKE_LORA_REVISION)
1017
+ lora_model.train(False)
1018
+ lora_model.to("cpu")
1019
+
1020
+ def _score_on(dev: Any) -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
1021
+ def _run(rgbs: list[Image.Image]) -> list[dict[str, Any]]:
1022
+ inputs = {k: v.to(dev) for k, v in processor(images=rgbs, return_tensors="pt").items()}
1023
+ with torch.inference_mode():
1024
+ probs = (lora_model(**inputs).logits / 2.0).softmax(-1)[:, 1].cpu().tolist()
1025
+ return [{"pred_ai": float(p_val) >= 0.5, "pred_provider": None} for p_val in probs]
1026
+
1027
+ return _run
1028
+
1029
+ def _move_to_acc() -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
1030
+ dev = _select_torch_accelerator()
1031
+ lora_model.to(dev)
1032
+ return _score_on(dev)
1033
+
1034
+ return _evaluate_rgb_detector(
1035
+ samples,
1036
+ rgb_cache,
1037
+ score_cpu_batch=_score_on(torch.device("cpu")),
1038
+ make_batch_scorer=_move_to_acc,
1039
+ model_id="bendavidsteel/openfake-swinv2-base + textgraphics-lora",
1040
+ revision=f"{OPENFAKE_BASE_REVISION[:10]}+{OPENFAKE_LORA_REVISION[:10]}",
1041
+ runtime="PyTorch + PEFT",
1042
+ weights_mb=lora_mb,
1043
+ license_id="cc-by-nc-4.0",
1044
+ tier="frontier",
1045
  )
1046
 
1047
 
1048
+ def evaluate_smaug(
1049
+ samples: list[SampleItem],
1050
+ rgb_cache: list[Image.Image] | None = None,
1051
+ ) -> dict[str, Any]:
1052
+ """Evaluate `abacusai/Smaug-AI-Image-Detector` (EmbeddingGemma-2 439M, 3x 528px crops)."""
1053
+ import torch
1054
+ from huggingface_hub import snapshot_download
1055
+ from sentence_transformers import SentenceTransformer
1056
+
1057
+ snap = Path(snapshot_download(SMAUG_REPO, revision=SMAUG_REVISION))
1058
+ weights_bytes = sum(p.stat().st_size for p in snap.rglob("*.safetensors") if p.is_file())
1059
+ weights_mb = round(weights_bytes / (1024 * 1024), 2)
1060
+
1061
+ st = SentenceTransformer(
1062
+ str(snap),
1063
+ device="cpu",
1064
+ config_kwargs={"audio_config": None},
1065
+ model_kwargs={"torch_dtype": torch.float32},
1066
+ )
1067
+ ip = st[0].processor.image_processor
1068
+ ip.max_soft_tokens = ip.image_seq_length = 140
1069
+
1070
+ def _score_batch(rgbs: list[Image.Image]) -> list[dict[str, Any]]:
1071
+ flat = [{"image": crop} for rgb in rgbs for crop in _smaug_views(rgb)]
1072
+ with torch.inference_mode():
1073
+ logits = st.encode(flat, batch_size=16, convert_to_tensor=True, show_progress_bar=False).float()
1074
+ probs = torch.sigmoid(logits.view(len(rgbs), 3).mean(dim=1)).cpu().tolist()
1075
+ return [{"pred_ai": float(p_val) >= 0.5, "pred_provider": None} for p_val in probs]
1076
+
1077
+ def _move_to_acc() -> Callable[[list[Image.Image]], list[dict[str, Any]]]:
1078
+ dev = _select_torch_accelerator()
1079
+ if dev.type == "cuda":
1080
+ st.to(device=dev, dtype=torch.bfloat16)
1081
+ elif dev.type != "cpu":
1082
+ st.to(device=dev)
1083
+ return _score_batch
1084
+
1085
+ return _evaluate_rgb_detector(
1086
+ samples,
1087
+ rgb_cache,
1088
+ score_cpu_batch=_score_batch,
1089
+ make_batch_scorer=_move_to_acc,
1090
+ batch_size=8,
1091
+ latency_count=SMAUG_CPU_LATENCY_COUNT,
1092
+ model_id=SMAUG_REPO,
1093
+ revision=SMAUG_REVISION,
1094
+ runtime="PyTorch + SentenceTransformers",
1095
+ weights_mb=weights_mb,
1096
+ license_id="apache-2.0",
1097
+ tier="frontier",
1098
+ )
1099
+
1100
+
1101
+ def _summarize_ai_subset(
1102
+ ai_pairs: list[tuple[SampleItem, dict[str, Any]]],
1103
+ *,
1104
+ non_ai_fp: int,
1105
+ ) -> dict[str, Any]:
1106
+ per_prov: dict[str, dict[str, int]] = {}
1107
+ tp = 0
1108
+ exact_provider_hits = 0
1109
+ total = len(ai_pairs)
1110
+
1111
+ for item, pred in ai_pairs:
1112
+ flagged = bool(pred["pred_ai"])
1113
+ prov = item.expected_provider or "unknown"
1114
+ bucket = per_prov.setdefault(prov, {"count": 0, "detected_ai": 0, "exact_provider": 0})
1115
+ bucket["count"] += 1
1116
+ if flagged:
1117
+ tp += 1
1118
+ bucket["detected_ai"] += 1
1119
+ if pred["pred_provider"] is not None and pred["pred_provider"] == item.expected_provider:
1120
+ exact_provider_hits += 1
1121
+ bucket["exact_provider"] += 1
1122
+
1123
+ precision = (tp / (tp + non_ai_fp) * 100.0) if (tp + non_ai_fp) else 0.0
1124
+ recall = (tp / total * 100.0) if total else 0.0
1125
+ attr_acc = (exact_provider_hits / total * 100.0) if total else 0.0
1126
+ attr_when_classified = (exact_provider_hits / tp * 100.0) if tp else 0.0
1127
+ per_provider_summary = {
1128
+ prov: {
1129
+ "count": b["count"],
1130
+ "detected_ai": b["detected_ai"],
1131
+ "exact_provider": b["exact_provider"],
1132
+ "recall_pct": round(b["detected_ai"] / b["count"] * 100.0, 1),
1133
+ "attribution_pct": round(b["exact_provider"] / b["count"] * 100.0, 1),
1134
+ }
1135
+ for prov, b in sorted(per_prov.items())
1136
+ }
1137
+ return {
1138
+ "total": total,
1139
+ "tp": tp,
1140
+ "recall_pct": round(recall, 1),
1141
+ "precision_pct": round(precision, 1),
1142
+ "provider_attribution_pct": round(attr_acc, 1),
1143
+ "provider_attribution_when_classified_pct": round(attr_when_classified, 1),
1144
+ "per_provider": per_provider_summary,
1145
+ }
1146
+
1147
+
1148
  def _summarize_predictions(
1149
  *,
1150
  model_id: str,
 
1155
  median_cpu_ms: float,
1156
  samples: list[SampleItem],
1157
  preds: list[dict[str, Any]],
1158
+ tier: str = "frontier",
1159
  ) -> dict[str, Any]:
1160
  non_ai_by_cat: dict[str, list[bool]] = {}
1161
+ native_ai_pairs: list[tuple[SampleItem, dict[str, Any]]] = []
1162
+ arena_ai_pairs: list[tuple[SampleItem, dict[str, Any]]] = []
 
 
1163
 
1164
  for item, pred in zip(samples, preds, strict=True):
1165
+ if item.regime == "non_ai":
1166
+ non_ai_by_cat.setdefault(item.category, []).append(bool(pred["pred_ai"]))
1167
+ elif item.regime == "native":
1168
+ native_ai_pairs.append((item, pred))
1169
+ elif item.regime == "arena":
1170
+ arena_ai_pairs.append((item, pred))
 
 
 
 
 
 
1171
  else:
1172
+ raise ValueError(f"Unsupported sample regime: {item.regime!r}")
 
 
 
 
 
 
 
 
 
 
 
1173
 
1174
+ non_ai_total = sum(len(flags) for flags in non_ai_by_cat.values())
1175
+ fp = sum(sum(flags) for flags in non_ai_by_cat.values())
1176
+ fpr = (fp / non_ai_total * 100.0) if non_ai_total else 0.0
1177
  slice_fprs = {cat: round(sum(flags) / len(flags) * 100.0, 1) for cat, flags in non_ai_by_cat.items()}
1178
+
1179
+ all_ai = _summarize_ai_subset(native_ai_pairs + arena_ai_pairs, non_ai_fp=fp)
1180
+ native_ai = _summarize_ai_subset(native_ai_pairs, non_ai_fp=fp)
1181
+ arena_ai = _summarize_ai_subset(arena_ai_pairs, non_ai_fp=fp)
1182
+
 
 
 
 
 
1183
  return {
1184
  "model_id": model_id,
1185
+ "tier": tier,
1186
  "revision": revision,
1187
  "license": license_id,
1188
  "weights_mb": weights_mb,
 
1192
  "non_ai_fp": fp,
1193
  "non_ai_fpr_pct": round(fpr, 2),
1194
  "slice_fpr_pct": slice_fprs,
1195
+ "ai_total": all_ai["total"],
1196
+ "ai_tp": all_ai["tp"],
1197
+ "ai_recall_pct": all_ai["recall_pct"],
1198
+ "precision_pct": all_ai["precision_pct"],
1199
+ "provider_attribution_pct": all_ai["provider_attribution_pct"],
1200
+ "provider_attribution_when_classified_pct": all_ai["provider_attribution_when_classified_pct"],
1201
+ "per_provider": all_ai["per_provider"],
1202
+ "native_ai": native_ai,
1203
+ "arena_ai": arena_ai,
1204
  }
1205
 
1206
 
1207
  def format_markdown_table(results: list[dict[str, Any]]) -> str:
1208
+ """Format head-to-head results across non-AI traps, native exports, and transcoded arena images."""
1209
+ n_non_ai = results[0]["non_ai_total"] if results else 0
1210
+ n_native = results[0]["native_ai"]["total"] if results else 0
1211
+ n_arena = results[0]["arena_ai"]["total"] if results else 0
1212
+
1213
  header = (
1214
+ f"| Model | License | Size | CPU ms | Non-AI FPR (`{n_non_ai}`) | Photo / UI / Diag / Rcpt FPR "
1215
+ f"| Native AI Recall (`{n_native}`) | Native Attribution (`{n_native}`) "
1216
+ f"| Arena AI Recall (`{n_arena}`) | Arena Attribution (`{n_arena}`) | Precision |"
1217
  )
1218
+ sep = "|---|---|---:|---:|---:|---|---:|---:|---:|---:|---:|"
1219
  lines = [header, sep]
1220
  for r in results:
1221
  sf = r["slice_fpr_pct"]
1222
+ nat = r["native_ai"]
1223
+ arn = r["arena_ai"]
1224
+ slice_str = (
1225
+ f"{sf.get('camera_photo', 0.0):.0f}% / {sf.get('ui_screenshot', 0.0):.0f}% / "
1226
+ f"{sf.get('science_diagram', 0.0):.0f}% / {sf.get('receipt_invoice', 0.0):.0f}%"
1227
+ )
1228
  lines.append(
1229
+ f"| `{r['model_id']}` | `{r['license']}` | {r['weights_mb']:.1f} MB | {r['median_cpu_ms']:.0f} ms "
1230
  f"| {r['non_ai_fpr_pct']:.1f}% ({r['non_ai_fp']}/{r['non_ai_total']}) "
1231
+ f"| {slice_str} "
1232
+ f"| {nat['recall_pct']:.1f}% "
1233
+ f"| {nat['provider_attribution_pct']:.1f}% "
1234
+ f"| {arn['recall_pct']:.1f}% "
1235
+ f"| {arn['provider_attribution_pct']:.1f}% "
1236
+ f"| {r['precision_pct']:.1f}% |"
 
1237
  )
1238
  return "\n".join(lines)
1239
 
1240
 
1241
+ def _format_buckets(items: list[SampleItem]) -> dict[str, dict[str, int]]:
1242
+ out: dict[str, dict[str, int]] = {}
1243
+ for item in items:
1244
+ prov = item.expected_provider or item.category
1245
+ fmap = out.setdefault(prov, {})
1246
+ key = f"{item.container_format}:{item.width}x{item.height}"
1247
+ fmap[key] = fmap.get(key, 0) + 1
1248
+ return out
1249
+
1250
+
1251
  def main() -> int:
1252
+ from huggingface_hub import HfApi, HfFileSystem
1253
 
1254
  parser = argparse.ArgumentParser(description=__doc__)
1255
  parser.add_argument("--json-out", type=Path, help="Optional path to write full JSON benchmark report.")
1256
  parser.add_argument("--raiw-only", action="store_true", help="Run only wiltodelta/raiw-source-classify.")
1257
+ parser.add_argument(
1258
+ "--tier",
1259
+ choices=("all", "frontier", "legacy"),
1260
+ default="all",
1261
+ help="Baseline tier to run alongside wiltodelta/raiw-source-classify (default: all).",
1262
+ )
1263
  args = parser.parse_args()
1264
 
1265
  logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
1266
  fs = HfFileSystem()
1267
+ api = HfApi()
1268
+
1269
+ all_samples = load_all_samples(fs, api)
1270
+ non_ai = [s for s in all_samples if s.regime == "non_ai"]
1271
+ native_ai = [s for s in all_samples if s.regime == "native"]
1272
+ arena_ai = [s for s in all_samples if s.regime == "arena"]
1273
+ log.info(
1274
+ "Evaluating %s total images (%s non-AI + %s native AI + %s transcoded arena AI)",
1275
+ len(all_samples),
1276
+ len(non_ai),
1277
+ len(native_ai),
1278
+ len(arena_ai),
1279
+ )
1280
 
1281
  results: list[dict[str, Any]] = [evaluate_raiw(all_samples)]
1282
+ _release_memory()
1283
+
1284
  if not args.raiw_only:
1285
+ with ThreadPoolExecutor(max_workers=8) as pool:
1286
+ rgb_cache = list(pool.map(lambda s: _decode_rgb(s.raw_bytes), all_samples))
1287
+
1288
+ evaluators: list[tuple[str, Callable[[], dict[str, Any]]]] = []
1289
+ if args.tier in ("all", "frontier"):
1290
+ evaluators.extend(
1291
+ [
1292
+ (SMAUG_REPO, lambda: evaluate_smaug(all_samples, rgb_cache)),
1293
+ (
1294
+ COMMFOR_DEEPFAKEDET_SPEC.repo_id,
1295
+ lambda: evaluate_hf_binary_detector(COMMFOR_DEEPFAKEDET_SPEC, all_samples, rgb_cache),
1296
+ ),
1297
+ *[
1298
+ (
1299
+ vit_spec.repo_id,
1300
+ lambda vs=vit_spec: evaluate_timm_vit_detector(vs, all_samples, rgb_cache),
1301
+ )
1302
+ for vit_spec in FRONTIER_VIT_DETECTORS
1303
+ ],
1304
+ (DETECTRA_OPENFAKE_REPO, lambda: evaluate_detectra_openfake(all_samples, rgb_cache)),
1305
+ (
1306
+ "bendavidsteel/openfake-swinv2-base + attribution-head",
1307
+ lambda: evaluate_openfake_attribution(all_samples, rgb_cache),
1308
+ ),
1309
+ (
1310
+ "bendavidsteel/openfake-swinv2-base + textgraphics-lora",
1311
+ lambda: evaluate_openfake_textgraphics_lora(all_samples, rgb_cache),
1312
+ ),
1313
+ ]
1314
+ )
1315
+ if args.tier in ("all", "legacy"):
1316
+ evaluators.extend(
1317
+ [
1318
+ (spec.repo_id, lambda sp=spec: evaluate_hf_binary_detector(sp, all_samples, rgb_cache))
1319
+ for spec in LEGACY_BINARY_DETECTORS
1320
+ ]
1321
+ )
1322
+
1323
+ for label, run_eval in evaluators:
1324
+ log.info("Evaluating %s ...", label)
1325
+ results.append(run_eval())
1326
+ _release_memory()
1327
+ del rgb_cache
1328
+ _release_memory()
1329
 
1330
  md_table = format_markdown_table(results)
1331
  sys.stdout.write("\n" + md_table + "\n")
1332
 
1333
  if args.json_out is not None:
1334
+ native_formats = _format_buckets(native_ai)
1335
+ arena_formats = _format_buckets(arena_ai)
 
 
 
 
1336
  payload = {
1337
  "non_ai_count": len(non_ai),
1338
+ "native_ai_count": len(native_ai),
1339
+ "arena_ai_count": len(arena_ai),
1340
+ "native_ai_providers": {p: sum(m.values()) for p, m in sorted(native_formats.items())},
1341
+ "native_ai_formats": native_formats,
1342
+ "arena_ai_providers": {p: sum(m.values()) for p, m in sorted(arena_formats.items())},
1343
+ "arena_ai_formats": arena_formats,
1344
  "results": results,
1345
  }
1346
  args.json_out.parent.mkdir(parents=True, exist_ok=True)