Wrapper code (speech-spoof-bench)
Browse files- .gitignore +14 -0
- LICENSE.source +23 -0
- _net.py +82 -0
- cmp_scores.py +22 -0
- implementation-notes.md +58 -0
- meta.yaml +47 -0
- rawnetlite.py +49 -0
- source_parity.py +42 -0
- subset_eer.py +27 -0
- sweep.py +39 -0
- test_rawnetlite.py +78 -0
.gitignore
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hardware-specific TensorRT artifacts — rebuild per GPU with:
|
| 2 |
+
# CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=<gpu> python trt_<module>.py export
|
| 3 |
+
*.plan
|
| 4 |
+
.trt_timing_*.cache
|
| 5 |
+
|
| 6 |
+
# Python
|
| 7 |
+
__pycache__/
|
| 8 |
+
*.py[cod]
|
| 9 |
+
.pytest_cache/
|
| 10 |
+
|
| 11 |
+
# Caches & logs
|
| 12 |
+
.cache/
|
| 13 |
+
*.log
|
| 14 |
+
*.out
|
LICENSE.source
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Upstream: github.com/adipiz99/RawNetLite @ fcb0fe8d0f40146d110e322c00a0723ba27d5520 (code: RawNetLite.py, audio_preprocessor.py vendored as _net.py; weights: models/triple_cross_domain_focal_rawnet_lite.pt). Its LICENSE file, verbatim:
|
| 2 |
+
|
| 3 |
+
MIT License
|
| 4 |
+
|
| 5 |
+
Copyright (c) 2025 Andrea Di Pierno
|
| 6 |
+
|
| 7 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 8 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 9 |
+
in the Software without restriction, including without limitation the rights
|
| 10 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 11 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 12 |
+
furnished to do so, subject to the following conditions:
|
| 13 |
+
|
| 14 |
+
The above copyright notice and this permission notice shall be included in all
|
| 15 |
+
copies or substantial portions of the Software.
|
| 16 |
+
|
| 17 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 18 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 19 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 20 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 21 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 22 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 23 |
+
SOFTWARE.
|
_net.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Vendored from github.com/adipiz99/RawNetLite @ fcb0fe8d0f40146d110e322c00a0723ba27d5520 (MIT, (c) 2025 Andrea Di Pierno):
|
| 2 |
+
RawNetLite.py (ResBlock, RawNetLite) and audio_preprocessor.py (preprocess_audio).
|
| 3 |
+
|
| 4 |
+
Edits for the Arena wrapper:
|
| 5 |
+
1. RawNetLite.forward_logit = forward without the final torch.sigmoid (returns the fc2 pre-activation, the
|
| 6 |
+
spoof logit); forward() is kept verbatim (sigmoid(spoof logit) = P(fake)). Docstrings dropped.
|
| 7 |
+
2. prep_np = numpy port of preprocess_audio for an already-mono 16 kHz clip: divide by max |x| over the WHOLE clip,
|
| 8 |
+
then keep the first 48000 samples (3 s) / right zero-pad. No resampling (the Arena delivers 16 kHz =
|
| 9 |
+
the authors' target_sr). A silent/empty clip (max |x| = 0, where the source would give NaN) -> zeros.
|
| 10 |
+
"""
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn as nn
|
| 14 |
+
|
| 15 |
+
TARGET_SR = 16000
|
| 16 |
+
TARGET_SAMPLES = 48000 # audio_preprocessor.py: target_sec=3.0 at 16 kHz (shorter than the Arena 4 s cap)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def prep_np(wav, max_len=TARGET_SAMPLES):
|
| 20 |
+
wav = np.asarray(wav, dtype=np.float32).reshape(-1)
|
| 21 |
+
peak = np.abs(wav).max() if wav.size else 0.0
|
| 22 |
+
if peak > 0:
|
| 23 |
+
wav = wav / peak
|
| 24 |
+
out = np.zeros(max_len, dtype=np.float32)
|
| 25 |
+
n = min(wav.shape[0], max_len)
|
| 26 |
+
out[:n] = wav[:n]
|
| 27 |
+
return out
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class ResBlock(nn.Module):
|
| 31 |
+
def __init__(self, channels):
|
| 32 |
+
super(ResBlock, self).__init__()
|
| 33 |
+
self.conv1 = nn.Conv1d(channels, channels, kernel_size=3, padding=1)
|
| 34 |
+
self.bn1 = nn.BatchNorm1d(channels)
|
| 35 |
+
self.relu = nn.ReLU()
|
| 36 |
+
self.conv2 = nn.Conv1d(channels, channels, kernel_size=3, padding=1)
|
| 37 |
+
self.bn2 = nn.BatchNorm1d(channels)
|
| 38 |
+
|
| 39 |
+
def forward(self, x):
|
| 40 |
+
residual = x
|
| 41 |
+
x = self.relu(self.bn1(self.conv1(x)))
|
| 42 |
+
x = self.bn2(self.conv2(x))
|
| 43 |
+
return self.relu(x + residual)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class RawNetLite(nn.Module):
|
| 47 |
+
def __init__(self):
|
| 48 |
+
super(RawNetLite, self).__init__()
|
| 49 |
+
self.conv_pre = nn.Conv1d(1, 64, kernel_size=3, stride=1, padding=1)
|
| 50 |
+
self.bn_pre = nn.BatchNorm1d(64)
|
| 51 |
+
self.relu = nn.ReLU()
|
| 52 |
+
|
| 53 |
+
self.resblock1 = ResBlock(64)
|
| 54 |
+
self.resblock2 = ResBlock(64)
|
| 55 |
+
self.resblock3 = ResBlock(64)
|
| 56 |
+
|
| 57 |
+
self.pool = nn.AdaptiveAvgPool1d(64) # Sequence reduction for GRU
|
| 58 |
+
|
| 59 |
+
self.gru = nn.GRU(input_size=64, hidden_size=128, num_layers=1,
|
| 60 |
+
batch_first=True, bidirectional=True)
|
| 61 |
+
|
| 62 |
+
self.fc1 = nn.Linear(128 * 2, 64)
|
| 63 |
+
self.fc2 = nn.Linear(64, 1)
|
| 64 |
+
|
| 65 |
+
def forward_logit(self, x): # edit 1
|
| 66 |
+
# x: [B, 1, T]
|
| 67 |
+
x = self.relu(self.bn_pre(self.conv_pre(x))) # [B, 64, T]
|
| 68 |
+
x = self.resblock1(x)
|
| 69 |
+
x = self.resblock2(x)
|
| 70 |
+
x = self.resblock3(x)
|
| 71 |
+
x = self.pool(x) # [B, 64, 64]
|
| 72 |
+
|
| 73 |
+
x = x.transpose(1, 2) # [B, 64, 64] -> [B, seq, feat]
|
| 74 |
+
output, _ = self.gru(x) # [B, 64, 256]
|
| 75 |
+
x = output[:, -1, :] # Last step -> [B, 256]
|
| 76 |
+
|
| 77 |
+
x = self.fc1(x) # [B, 64]
|
| 78 |
+
x = self.fc2(x) # [B, 1]
|
| 79 |
+
return x
|
| 80 |
+
|
| 81 |
+
def forward(self, x):
|
| 82 |
+
return torch.sigmoid(self.forward_logit(x))
|
cmp_scores.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# cmp_scores.py A/scores.txt B/scores.txt -> n, Spearman, max |diff|
|
| 2 |
+
import sys
|
| 3 |
+
import numpy as np
|
| 4 |
+
from scipy.stats import spearmanr
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def read(p):
|
| 8 |
+
d = {}
|
| 9 |
+
for line in open(p):
|
| 10 |
+
parts = line.split()
|
| 11 |
+
if len(parts) >= 2:
|
| 12 |
+
try:
|
| 13 |
+
d[parts[0]] = float(parts[-1])
|
| 14 |
+
except ValueError:
|
| 15 |
+
pass
|
| 16 |
+
return d
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
a, b = read(sys.argv[1]), read(sys.argv[2])
|
| 20 |
+
k = sorted(set(a) & set(b))
|
| 21 |
+
x, y = np.array([a[i] for i in k]), np.array([b[i] for i in k])
|
| 22 |
+
print(f"n={len(k)} spearman={spearmanr(x, y).correlation:.6f} max_abs_diff={np.max(np.abs(x - y)):.4g}")
|
implementation-notes.md
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# RawNetLite — implementation notes
|
| 2 |
+
|
| 3 |
+
Plan: `speech-spoof-bench/docs/plans/2026-11-09-thirteen-model-batch-arena-submission.md` (model #13).
|
| 4 |
+
|
| 5 |
+
## Source
|
| 6 |
+
- github.com/adipiz99/RawNetLite @ `fcb0fe8d0f40146d110e322c00a0723ba27d5520` (MIT, `LICENSE.source`), clone in
|
| 7 |
+
`_ckpt_dl/gh__adipiz99__RawNetLite/`. Vendored into `_net.py`: `RawNetLite.py` (ResBlock, RawNetLite) and
|
| 8 |
+
`audio_preprocessor.py:preprocess_audio`. Read but not needed: tester.py, trainer.py, *_dataset.py, Mixed_dataset.py.
|
| 9 |
+
- Checkpoint `models/triple_cross_domain_focal_rawnet_lite.pt` (963.8 KB, sha256
|
| 10 |
+
`6154d91f34f0a49d58e30bac602915f6b6853419419804be63b5274603891a9b`): plain state_dict, 61 tensors, loads with
|
| 11 |
+
`weights_only=True`. Symlinked here. (The repo also has rawnet_lite / cross_domain(_focal) / augmented_triple variants.)
|
| 12 |
+
- Paper: arXiv 2504.20923 (Di Pierno, Guarnera, Allegra, Battiato; IJCNN 2025 VERIMEDIA workshop).
|
| 13 |
+
|
| 14 |
+
## Wrapper (`rawnetlite.py:RawNetLite`, `_net.py`)
|
| 15 |
+
- Preprocessing = `preprocess_audio` for 16 kHz mono input: `x / max|x|` over the whole clip, then first 48000 samples
|
| 16 |
+
(3 s, authors' fixed length; under the 4 s cap so kept), right zero-pad. Authors' target_sr is 16 kHz → no resampling
|
| 17 |
+
ever. Silent/empty clip: source divides by 0 (NaN) → wrapper feeds zeros.
|
| 18 |
+
- Net input [B, 1, 48000] (DataLoader stacks the [1, 48000] tensors). Output: source applies sigmoid → P(fake)
|
| 19 |
+
(tester.py: probs > 0.5 = fake, EER with pos_label=1; label 0 = real, 1 = fake). Wrapper returns **−logit(fake)**
|
| 20 |
+
(fc2 output before the sigmoid, negated): identical ranking to 1 − P(fake), no saturation ties.
|
| 21 |
+
- Strict load: `load_state_dict(strict=True)`, 0 missing / 0 unexpected, all 61 tensors used (no allow-list needed).
|
| 22 |
+
Params **240,769** trainable (+ BN running stats). FP32, TF32 off.
|
| 23 |
+
|
| 24 |
+
## Parity / tests
|
| 25 |
+
- `source_parity.py`: authors' `RawNetLite` (imported from the pinned clone) + `preprocess_audio` + tester.py's
|
| 26 |
+
`model(waveforms).squeeze()` per clip vs the wrapper batched, 16 J-SPAW_LA clips + a short and a long clip:
|
| 27 |
+
max |Δ P(fake)| 8.5e-8, max |Δ score| (vs −logit of the source P) 8.6e-7. Log: `logs_source_parity.txt`.
|
| 28 |
+
- `test_rawnetlite.py`: peak-norm-then-crop, zero-pad, silent clip, sha256, strict load, params + eval,
|
| 29 |
+
sigmoid(forward_logit) == forward, batch == single (atol 1e-4), determinism, scale invariance. 9 passed.
|
| 30 |
+
|
| 31 |
+
## Training data → trained_on
|
| 32 |
+
- Paper Tables I–III: FakeOrReal 25,600 + 25,600 train; "AVSpoof2021" 6,400 real + 6,400 fake train (test: 22,616 /
|
| 33 |
+
25,000 disjoint); CodecFake 6,400 + 6,400 train (triple / augmented models only). The released checkpoint is the
|
| 34 |
+
triple-domain focal model (README).
|
| 35 |
+
- "AVSpoof2021": README links asvspoof.org/index2021.html; the paper never names a track, describes it as containing
|
| 36 |
+
"replay attacks, TTS, and VC", and cites the ASVspoof 2019 overview (Todisco et al.) for it. ASVspoof 2021 only
|
| 37 |
+
released evaluation data, so the training clips come from 2021 eval → ASVspoof2021_LA **and** ASVspoof2021_DF marked
|
| 38 |
+
trained_on (track unknown). Note: their test real count 22,616 is one short of DF eval's 22,617 bona fide, but the
|
| 39 |
+
6,400 extra training reals rule out a DF-only pool — so the pool likely mixes tracks (PA too?). ASVspoof 2019 data
|
| 40 |
+
cannot be excluded (citation) but is not marked — maintainer decision.
|
| 41 |
+
- CodecFake: README links roger-tseng/CodecFake (Wu et al., VCTK-based); the paper cites Xie et al.'s Codecfake
|
| 42 |
+
(arXiv 2405.04880, VCTK + AISHELL-3). Neither is the Arena's CD-ADD (Li et al., arXiv 2404.04904, LibriTTS-based codec
|
| 43 |
+
TTS; provenance `CD-ADD: {parts: [train, test]}`) → no CD-ADD overlap. FakeOrReal is not an Arena set.
|
| 44 |
+
- Eval-subset EERs with this wrapper (`subset_eer.py 2000 <DS>`, shard 0, seed 0): ASVspoof2019_LA 13.82%,
|
| 45 |
+
ASVspoof2021_LA 20.24%, ASVspoof2021_DF 21.25% (paper AVSpoof2021 test: 19.96%).
|
| 46 |
+
|
| 47 |
+
## Sanity (PyTorch, RTX 5060 Ti idx 3)
|
| 48 |
+
- Full J-SPAW_LA (`run_pt.sh`, run at batch_size 64 before the sweep set it to 1; batch-invariant to ~1e-6): EER
|
| 49 |
+
**34.189%** (2397 trials, 0 skipped). Weak, as expected for this 0.24 M-param model; not inverted.
|
| 50 |
+
|
| 51 |
+
## Batch sweep (`sweep.py`, 48000-sample input)
|
| 52 |
+
| card | result | choice |
|
| 53 |
+
|---|---|---|
|
| 54 |
+
| RTX 5060 Ti (idx 3), 2026-11-09, run twice (idle-ish, then shared with the LaST-Net lane) | 238–240 utt/s @1; 165–200 utt/s @8–192 (best batched 200 @64); 256 OOM | `batch_size = 1` |
|
| 55 |
+
| RTX 4070 Ti SUPER (idx 2) | not swept (not assigned) | same 1 |
|
| 56 |
+
|
| 57 |
+
## TensorRT
|
| 58 |
+
Not attempted (plan: no TRT for the 1 MB row).
|
meta.yaml
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
system:
|
| 2 |
+
name: "RawNetLite"
|
| 3 |
+
slug: "rawnetlite"
|
| 4 |
+
description: >
|
| 5 |
+
RawNetLite (Di Pierno et al., IJCNN VERIMEDIA 2025): a lightweight end-to-end raw-waveform detector --
|
| 6 |
+
Conv1D(1->64) + BatchNorm + ReLU, three 1-D residual blocks (64 ch), adaptive average pooling to 64 steps,
|
| 7 |
+
a bidirectional GRU (128 per direction, last step), Linear(256,64) -> Linear(64,1). Released checkpoint
|
| 8 |
+
triple_cross_domain_focal_rawnet_lite.pt: trained with focal loss on a balanced mix of FakeOrReal,
|
| 9 |
+
"AVSpoof2021" (the authors' name for ASVspoof 2021) and CodecFake, no augmentation. Input as the authors'
|
| 10 |
+
audio_preprocessor.py: 16 kHz mono, peak-normalised (divide by max |x| of the whole clip), first 48000
|
| 11 |
+
samples (3 s, within the Arena 4 s cap), right zero-pad. score = -logit(fake) (the pre-sigmoid fc2 output
|
| 12 |
+
negated; the source's sigmoid(logit) is P(fake)). FP32 (TF32 off).
|
| 13 |
+
code: "https://github.com/adipiz99/RawNetLite/tree/fcb0fe8d0f40146d110e322c00a0723ba27d5520"
|
| 14 |
+
checkpoint: "https://huggingface.co/SpeechAntiSpoofingBenchmarks/RawNetLite/blob/7209f7f94f1c69bc0d2320e854914e0966b12bd4/triple_cross_domain_focal_rawnet_lite.pt"
|
| 15 |
+
params_millions: 0.240769 # counted from the loaded net (240,769 trainable; asserted in test_rawnetlite.py), 2026-11-09
|
| 16 |
+
paper:
|
| 17 |
+
arxiv_id: "2504.20923"
|
| 18 |
+
url: "https://arxiv.org/abs/2504.20923"
|
| 19 |
+
bibtex: |
|
| 20 |
+
@inproceedings{dipierno2025rawnetlite,
|
| 21 |
+
title = {End-to-end Audio Deepfake Detection from RAW Waveforms: a RawNet-Based Approach with Cross-Dataset Evaluation},
|
| 22 |
+
author = {Andrea Di Pierno and Luca Guarnera and Dario Allegra and Sebastiano Battiato},
|
| 23 |
+
booktitle = {International Joint Conference on Neural Networks (IJCNN) - VERIMEDIA Workshop},
|
| 24 |
+
year = {2025}
|
| 25 |
+
}
|
| 26 |
+
trained_on:
|
| 27 |
+
- SpeechAntiSpoofingBenchmarks/ASVspoof2021_LA
|
| 28 |
+
- SpeechAntiSpoofingBenchmarks/ASVspoof2021_DF
|
| 29 |
+
notes: >
|
| 30 |
+
RawNetLite = github.com/adipiz99/RawNetLite @ fcb0fe8d0f40146d110e322c00a0723ba27d5520 (MIT; RawNetLite.py and
|
| 31 |
+
audio_preprocessor.py vendored), checkpoint models/triple_cross_domain_focal_rawnet_lite.pt (sha256
|
| 32 |
+
6154d91f34f0a49d58e30bac602915f6b6853419419804be63b5274603891a9b; plain state_dict, 61 tensors, weights_only,
|
| 33 |
+
strict load, all used; re-hosted under MIT). Wrapper vs the authors' RawNetLite + preprocess_audio + tester.py
|
| 34 |
+
scoring on J-SPAW clips: max |d P(fake)| 8.5e-8. Deviations: score is -logit instead of the sigmoid output
|
| 35 |
+
(same ranking, avoids float saturation ties); a silent clip (source: division by zero -> NaN) is scored as
|
| 36 |
+
zeros; no resampling (authors' target is 16 kHz too). The authors' 3 s window is kept (shorter than the 4 s cap).
|
| 37 |
+
Training data (paper Tables I-III + README): FakeOrReal (25,600+25,600), "AVSpoof2021" 6,400+6,400 and
|
| 38 |
+
CodecFake 6,400+6,400 training clips. The paper never names the ASVspoof 2021 track and 2021 has no train
|
| 39 |
+
split (the training clips come from the released evaluation data), so both ASVspoof2021_LA and
|
| 40 |
+
ASVspoof2021_DF are marked trained_on (n/a*). Its citation for "AVSpoof2021" is the ASVspoof 2019 overview
|
| 41 |
+
paper and it mentions replay attacks, so ASVspoof 2019 LA/PA or 2021 PA data cannot be excluded either
|
| 42 |
+
(not marked). CodecFake is linked to roger-tseng/CodecFake (Wu et al., VCTK-based) in the README and cited as
|
| 43 |
+
Xie et al.'s Codecfake in the paper; neither is the Arena's CD-ADD (Li et al., LibriTTS-based), and FakeOrReal
|
| 44 |
+
is not an Arena set -> no further entries. Self-reported EER% (paper Table XII, this checkpoint): FoR 2.66 /
|
| 45 |
+
AVSpoof2021 19.96 / CodecFake 21.13 / triple-domain test 16.44. 2000-clip eval subsets with this wrapper:
|
| 46 |
+
ASVspoof2019_LA 13.82, ASVspoof2021_LA 20.24, ASVspoof2021_DF 21.25. J-SPAW_LA scored with this wrapper
|
| 47 |
+
(EER 34.19%).
|
rawnetlite.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
from speech_spoof_bench.model import AntiSpoofingModel
|
| 8 |
+
|
| 9 |
+
from _net import TARGET_SAMPLES, RawNetLite as RawNetLiteNet, prep_np
|
| 10 |
+
|
| 11 |
+
_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 12 |
+
# github.com/adipiz99/RawNetLite @ fcb0fe8d0f40146d110e322c00a0723ba27d5520, models/triple_cross_domain_focal_rawnet_lite.pt
|
| 13 |
+
# sha256 6154d91f34f0a49d58e30bac602915f6b6853419419804be63b5274603891a9b -- plain state_dict (61 tensors), MIT
|
| 14 |
+
_CKPT = os.path.join(_DIR, "triple_cross_domain_focal_rawnet_lite.pt")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def load_state(path=_CKPT):
|
| 18 |
+
return torch.load(path, map_location="cpu", weights_only=True)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def build_net(path=_CKPT):
|
| 22 |
+
net = RawNetLiteNet()
|
| 23 |
+
r = net.load_state_dict(load_state(path), strict=True) # all 61 tensors used, 0 missing / 0 unexpected
|
| 24 |
+
assert not r.missing_keys and not r.unexpected_keys
|
| 25 |
+
return net
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class RawNetLite(AntiSpoofingModel):
|
| 29 |
+
name = "RawNetLite"
|
| 30 |
+
expected_sample_rate = 16000
|
| 31 |
+
batch_size = 1 # sweep 2026-11-09 (48000-sample input, RTX 5060 Ti idx 3): 238 utt/s @1, ~172-183 @8-192 (4070 not swept)
|
| 32 |
+
|
| 33 |
+
def load(self) -> None:
|
| 34 |
+
self.device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 35 |
+
torch.backends.cudnn.allow_tf32 = False
|
| 36 |
+
torch.backends.cuda.matmul.allow_tf32 = False
|
| 37 |
+
self.net = build_net().to(self.device).eval()
|
| 38 |
+
|
| 39 |
+
@torch.no_grad()
|
| 40 |
+
def score_batch(self, audios, srs):
|
| 41 |
+
x = torch.from_numpy(np.stack([prep_np(a, TARGET_SAMPLES) for a in audios]))[:, None, :].to(self.device)
|
| 42 |
+
logit_fake = self.net.forward_logit(x)[:, 0].float()
|
| 43 |
+
# source: sigmoid(logit) = P(fake) (label 1 = fake); -logit is the same ranking without sigmoid saturation
|
| 44 |
+
return (-logit_fake).cpu().tolist()
|
| 45 |
+
|
| 46 |
+
def unload(self) -> None:
|
| 47 |
+
self.net = None
|
| 48 |
+
if torch.cuda.is_available():
|
| 49 |
+
torch.cuda.empty_cache()
|
source_parity.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# source_parity.py [SRC_DIR] -- authors' RawNetLite.RawNetLite + audio_preprocessor.preprocess_audio + tester.py scoring
|
| 2 |
+
# (outputs = model(waveforms).squeeze() = P(fake)) vs this wrapper on 16 J-SPAW_LA clips (+ a short and a long clip).
|
| 3 |
+
# Checkpoint loaded with weights_only=True instead of tester.py's plain torch.load. Compares -logit(P_fake) (wrapper
|
| 4 |
+
# score) with -logit of the source's sigmoid output, and the source's P(fake) with sigmoid(-wrapper score).
|
| 5 |
+
import io, os, sys
|
| 6 |
+
import numpy as np, pyarrow.parquet as pq, soundfile as sf, torch
|
| 7 |
+
|
| 8 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 9 |
+
sys.path.insert(0, HERE)
|
| 10 |
+
import rawnetlite as W
|
| 11 |
+
src = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "..", "_ckpt_dl", "gh__adipiz99__RawNetLite")
|
| 12 |
+
sys.path.append(src)
|
| 13 |
+
import RawNetLite as SRC # noqa: E402 (pinned reviewed clone)
|
| 14 |
+
from audio_preprocessor import preprocess_audio # noqa: E402
|
| 15 |
+
|
| 16 |
+
torch.backends.cudnn.allow_tf32 = False
|
| 17 |
+
torch.backends.cuda.matmul.allow_tf32 = False
|
| 18 |
+
dev = "cuda"
|
| 19 |
+
ref = SRC.RawNetLite().to(dev)
|
| 20 |
+
ref.load_state_dict(torch.load(W._CKPT, map_location="cpu", weights_only=True))
|
| 21 |
+
ref.eval()
|
| 22 |
+
m = W.RawNetLite(); m.load()
|
| 23 |
+
|
| 24 |
+
d = os.path.join(HERE, "..", "J-SPAW_LA", "data")
|
| 25 |
+
t = pq.read_table(os.path.join(d, "test-00000-of-00001.parquet"), columns=["audio"]).slice(0, 16).column("audio").to_pylist()
|
| 26 |
+
auds = [sf.read(io.BytesIO(x["bytes"]), dtype="float32")[0] for x in t]
|
| 27 |
+
auds.append(auds[0][:20000]); auds.append(np.concatenate([auds[1]] * 4))
|
| 28 |
+
ours = np.array(m.score_batch(auds, [16000] * len(auds)))
|
| 29 |
+
p_src = []
|
| 30 |
+
with torch.no_grad():
|
| 31 |
+
for a in auds:
|
| 32 |
+
w = preprocess_audio(torch.from_numpy(a)[None, :], 16000) # [1, 48000]
|
| 33 |
+
p_src.append(float(ref(w[None].to(dev)).squeeze())) # DataLoader adds the batch dim -> [1, 1, 48000]
|
| 34 |
+
p_src = np.array(p_src, dtype=np.float64)
|
| 35 |
+
p_ours = 1.0 / (1.0 + np.exp(ours)) # sigmoid(logit_fake), logit_fake = -score
|
| 36 |
+
print("lens", [len(a) for a in auds])
|
| 37 |
+
print("ours score (-logit_fake)", np.round(ours, 4))
|
| 38 |
+
print("source P(fake) ", np.round(p_src, 5))
|
| 39 |
+
print("max_abs_diff P(fake)", float(np.max(np.abs(p_ours - p_src))))
|
| 40 |
+
ok = (p_src > 1e-6) & (p_src < 1 - 1e-6)
|
| 41 |
+
print("max_abs_diff score where source P not saturated (n=%d)" % ok.sum(),
|
| 42 |
+
float(np.max(np.abs(ours[ok] - (-np.log(p_src[ok] / (1 - p_src[ok])))))) if ok.any() else "n/a")
|
subset_eer.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# subset_eer.py [N] [DATASET] -- quick EER on a fixed random eval subset (default ASVspoof2019_LA):
|
| 2 |
+
# N rows sampled uniformly without replacement (seed 0) from shard 0. Cheap sanity vs the paper's ASV19 EER.
|
| 3 |
+
import io, os, sys
|
| 4 |
+
import numpy as np, pyarrow.parquet as pq, soundfile as sf
|
| 5 |
+
|
| 6 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 7 |
+
sys.path.insert(0, HERE)
|
| 8 |
+
mod = __import__([m[:-3] for m in os.listdir(HERE) if m.startswith(("wst_graph_q8", "rawnetlite")) and m.endswith(".py") and not m.startswith("test_")][0])
|
| 9 |
+
Cls = getattr(mod, [n for n in dir(mod) if n.startswith(("WSTGraphQ8", "RawNetLite")) and isinstance(getattr(mod, n), type) and hasattr(getattr(mod, n), "score_batch")][0])
|
| 10 |
+
N = int(sys.argv[1]) if len(sys.argv) > 1 else 2000
|
| 11 |
+
DS = sys.argv[2] if len(sys.argv) > 2 else "ASVspoof2019_LA"
|
| 12 |
+
t = pq.read_table(os.path.join(HERE, "..", DS, "data", sorted(f for f in os.listdir(os.path.join(HERE, "..", DS, "data")) if f.startswith("test-"))[0]), columns=["audio", "label"])
|
| 13 |
+
idx = np.sort(np.random.default_rng(0).choice(t.num_rows, N, replace=False))
|
| 14 |
+
t = t.take(idx).to_pylist()
|
| 15 |
+
m = Cls(); m.load()
|
| 16 |
+
scores, labels = [], []
|
| 17 |
+
B = 16
|
| 18 |
+
for k in range(0, len(t), B):
|
| 19 |
+
rows = t[k:k + B]
|
| 20 |
+
scores += m.score_batch([sf.read(io.BytesIO(r["audio"]["bytes"]), dtype="float32")[0] for r in rows], [16000] * len(rows))
|
| 21 |
+
labels += [r["label"] for r in rows]
|
| 22 |
+
s, y = np.array(scores), np.array(labels)
|
| 23 |
+
bona, spoof = s[y == 0], s[y == 1]
|
| 24 |
+
th = np.sort(s)
|
| 25 |
+
frr = np.array([(bona < x).mean() for x in th]); far = np.array([(spoof >= x).mean() for x in th])
|
| 26 |
+
i = np.argmin(np.abs(frr - far))
|
| 27 |
+
print(f"{Cls.__name__} {DS} subset n={len(s)} bona={len(bona)} spoof={len(spoof)} EER~{100 * (frr[i] + far[i]) / 2:.3f}%")
|
sweep.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# sweep.py -- batch-size throughput sweep (48000-sample inputs = the model's fixed 3 s window)
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import time
|
| 4 |
+
from statistics import median
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
import rawnetlite as R
|
| 8 |
+
|
| 9 |
+
_CUT = 48000
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def time_batch_sizes():
|
| 13 |
+
print("device", torch.cuda.get_device_name(), "| other GPU processes during the sweep make it unreliable (check nvidia-smi)")
|
| 14 |
+
m = R.RawNetLite(); m.load()
|
| 15 |
+
results = {}
|
| 16 |
+
for bs in [1, 8, 16, 32, 64, 96, 128, 192, 256]:
|
| 17 |
+
audios = [np.random.randn(_CUT).astype(np.float32) * 0.1 for _ in range(bs)]
|
| 18 |
+
srs = [16000] * bs
|
| 19 |
+
try:
|
| 20 |
+
m.score_batch(audios, srs); torch.cuda.synchronize()
|
| 21 |
+
reps = []
|
| 22 |
+
for _ in range(5):
|
| 23 |
+
torch.cuda.synchronize(); t0 = time.perf_counter()
|
| 24 |
+
m.score_batch(audios, srs)
|
| 25 |
+
torch.cuda.synchronize(); reps.append(time.perf_counter() - t0)
|
| 26 |
+
med = median(reps); results[bs] = bs / med
|
| 27 |
+
print(f"bs={bs:3d} median={med*1000:8.3f} ms utt/s={bs/med:8.2f} peak_mem={torch.cuda.max_memory_allocated()/2**30:.2f} GiB")
|
| 28 |
+
except RuntimeError as e: # TorchScript wraps CUDA OOM in a plain RuntimeError
|
| 29 |
+
if "out of memory" not in str(e):
|
| 30 |
+
raise
|
| 31 |
+
print(f"bs={bs:3d} OOM"); torch.cuda.empty_cache()
|
| 32 |
+
break
|
| 33 |
+
m.unload()
|
| 34 |
+
best = max(results, key=results.get)
|
| 35 |
+
print(f"best batch_size = {best} (utt/s={results[best]:.2f})")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
if __name__ == "__main__":
|
| 39 |
+
time_batch_sizes()
|
test_rawnetlite.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import io
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pytest
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
import _net
|
| 10 |
+
import rawnetlite as R
|
| 11 |
+
|
| 12 |
+
SHA256 = "6154d91f34f0a49d58e30bac602915f6b6853419419804be63b5274603891a9b"
|
| 13 |
+
N_PARAMS = 240_769 # trainable parameters (BN running stats / num_batches_tracked are buffers)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def test_prep_peak_norm_then_prefix():
|
| 17 |
+
x = np.concatenate([np.full(48000, 0.1, np.float32), np.array([-0.5], np.float32), np.zeros(999, np.float32)])
|
| 18 |
+
out = _net.prep_np(x) # peak over the WHOLE clip (0.5 lies beyond the 3 s crop)
|
| 19 |
+
assert out.shape == (48000,) and out.dtype == np.float32 and np.allclose(out, 0.2)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def test_prep_short_right_zero_pad():
|
| 23 |
+
x = np.linspace(-0.25, 0.5, 1001, dtype=np.float32)
|
| 24 |
+
out = _net.prep_np(x)
|
| 25 |
+
assert np.allclose(out[:1001], x / 0.5) and not out[1001:].any() and abs(out.max() - 1.0) < 1e-7
|
| 26 |
+
assert not _net.prep_np(np.zeros(5000, np.float32)).any() # silent clip -> zeros (source: NaN)
|
| 27 |
+
assert not _net.prep_np(np.zeros(0, np.float32)).any()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def test_checkpoint_sha256():
|
| 31 |
+
assert hashlib.sha256(open(R._CKPT, "rb").read()).hexdigest() == SHA256
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def test_strict_load_all_tensors_used():
|
| 35 |
+
sd = R.load_state()
|
| 36 |
+
assert len(sd) == 61
|
| 37 |
+
net = _net.RawNetLite()
|
| 38 |
+
assert set(net.state_dict()) == set(sd)
|
| 39 |
+
r = net.load_state_dict(sd, strict=True)
|
| 40 |
+
assert not r.missing_keys and not r.unexpected_keys
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@pytest.fixture(scope="module")
|
| 44 |
+
def model():
|
| 45 |
+
m = R.RawNetLite(); m.load()
|
| 46 |
+
yield m
|
| 47 |
+
m.unload()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_eval_and_params(model):
|
| 51 |
+
assert not any(mod.training for mod in model.net.modules())
|
| 52 |
+
assert sum(p.numel() for p in model.net.parameters()) == N_PARAMS
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_logit_matches_source_sigmoid(model):
|
| 56 |
+
x = torch.randn(3, 1, 48000, device=model.device) * 0.3
|
| 57 |
+
with torch.no_grad():
|
| 58 |
+
assert torch.allclose(torch.sigmoid(model.net.forward_logit(x)), model.net(x), atol=1e-6)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def test_shapes_and_batch_matches_single(model):
|
| 62 |
+
rng = np.random.default_rng(0)
|
| 63 |
+
a = [rng.standard_normal(n).astype(np.float32) * s for n, s in ((48000, 0.1), (20000, 0.5), (90000, 0.02), (3000, 0.3))]
|
| 64 |
+
batched = model.score_batch(a, [16000] * 4)
|
| 65 |
+
single = [model.score_batch([x], [16000])[0] for x in a]
|
| 66 |
+
assert len(batched) == 4 and all(np.isfinite(batched))
|
| 67 |
+
assert np.allclose(batched, single, atol=1e-4)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def test_deterministic(model):
|
| 71 |
+
a = [np.random.randn(40000).astype(np.float32) * 0.1 for _ in range(3)]
|
| 72 |
+
assert model.score_batch(a, [16000] * 3) == model.score_batch(a, [16000] * 3)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def test_scale_invariant(model):
|
| 76 |
+
x = np.random.default_rng(2).standard_normal(30000).astype(np.float32) * 0.1
|
| 77 |
+
s1, s2 = model.score_batch([x, 3 * x], [16000] * 2)
|
| 78 |
+
assert abs(s1 - s2) < 1e-4 # peak normalisation
|