korallll's picture
Wrapper code (speech-spoof-bench)
6332a34 verified
Raw History Blame Contribute Delete
12.5 kB
"""XLSR-Conformer + TCM (Temporal-Channel Modeling) network, ported from
https://github.com/ductuantruong/tcm_add (MIT) @ 726387eb731223a21d551476349f8f8f680a3375:
conformer.py (ConformerBlock + Head-Token Attention, verbatim below)
model.py (sinusoidal_embedding, MyConformer, SSLModel, Model)
Truong et al., "Temporal-Channel Modeling in Multi-head Self-Attention for Synthetic
Speech Detection", Interspeech 2024. Same network as benchmarks/DPDA-GA-TCM/_net.py
(dpda_ga re-used tcm_add's code; the only source differences are listed in edits 5, 7).
Hyper-parameters = main.py defaults: emb_size 144, heads 4, kernel_size 31,
num_encoders 4 (ff_mult 4, exp_fac 2); verified against the released checkpoint
shapes (strict load).
Edits (inference-only, numerically identical; parameter/module names unchanged so
the released state_dict loads strictly):
1. SSLModel builds the XLS-R 300M wav2vec2 architecture from the base
xlsr2_300m.pt model cfg merged onto the installed fairseq Wav2Vec2Config
(merge_with_parent; cross-version cfg-field fix, as W2V2-AASIST/_net.py),
instead of load_model_ensemble_and_task on a hard-coded path. The base
weights are not loaded -- the fine-tuned checkpoint overwrites every weight.
2. SSLModel.extract_feat: the `self.model.train()` call on a device/dtype
mismatch is removed (it would re-enable dropout at inference); the wrapper
moves the net to the device and calls .eval() itself.
3. Model.__init__ takes the hyper-parameters as keyword args (default = main.py)
instead of an argparse namespace; `device` args dropped (unused at inference).
4. MyConformer.forward: the class-token prepend
`torch.stack([torch.vstack((cls, x[i])) for i in range(len(x))])` is written as
`torch.cat([cls.expand(B, 1, E), x], 1)` (same tensor; the python loop over the
batch would freeze the batch size into a traced ONNX graph).
5. Model.forward / MyConformer.forward return only the (B, 2) logits (source returns
(out, list_of_attention_maps); main.py uses only out).
6. MyConformer.positional_emb is initialised to zeros instead of the slow python-list
sinusoid build: it is a frozen nn.Parameter stored in (and loaded from) the checkpoint.
7. (not an edit) Attention.forward keeps the source's commented-out attn_drop.
"""
import os
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops.layers.torch import Rearrange
from torch.nn.modules.transformer import _get_clones
_XLSR_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "xlsr2_300m.pt")
WINDOW = 66800 # source data_utils.Dataset_eval: self.cut = 66800 (fixed-length eval)
def pad(x: np.ndarray, max_len: int = WINDOW) -> np.ndarray:
"""source utils.pad: first max_len samples; shorter clips repeat-tiled."""
x = np.asarray(x, dtype=np.float32).reshape(-1)
x_len = x.shape[0]
if x_len >= max_len:
return x[:max_len]
num_repeats = int(max_len / x_len) + 1
return np.tile(x, (1, num_repeats))[:, :max_len][0]
# ============================ conformer.py ============================
def exists(val):
return val is not None
def default(val, d):
return val if exists(val) else d
def calc_same_padding(kernel_size):
pad = kernel_size // 2
return (pad, pad - (kernel_size + 1) % 2)
class Swish(nn.Module):
def forward(self, x):
return x * x.sigmoid()
class GLU(nn.Module):
def __init__(self, dim):
super().__init__()
self.dim = dim
def forward(self, x):
out, gate = x.chunk(2, dim=self.dim)
return out * gate.sigmoid()
class DepthWiseConv1d(nn.Module):
def __init__(self, chan_in, chan_out, kernel_size, padding):
super().__init__()
self.padding = padding
self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, groups=chan_in)
def forward(self, x):
x = F.pad(x, self.padding)
return self.conv(x)
class Scale(nn.Module):
def __init__(self, scale, fn):
super().__init__()
self.fn = fn
self.scale = scale
def forward(self, x, **kwargs):
return self.fn(x, **kwargs) * self.scale
class PreNorm(nn.Module):
def __init__(self, dim, fn):
super().__init__()
self.fn = fn
self.norm = nn.LayerNorm(dim)
def forward(self, x, **kwargs):
x = self.norm(x)
return self.fn(x, **kwargs)
class Attention(nn.Module):
# Head Token attention: https://arxiv.org/pdf/2210.05958.pdf
def __init__(self, dim, heads=8, dim_head=64, qkv_bias=False, dropout=0., proj_drop=0.):
super().__init__()
self.num_heads = heads
inner_dim = dim_head * heads
self.scale = dim_head ** -0.5
self.qkv = nn.Linear(dim, inner_dim * 3, bias=qkv_bias)
self.attn_drop = nn.Dropout(dropout)
self.proj = nn.Linear(inner_dim, dim)
self.proj_drop = nn.Dropout(proj_drop)
self.act = nn.GELU()
self.ht_proj = nn.Linear(dim_head, dim, bias=True)
self.ht_norm = nn.LayerNorm(dim_head)
self.pos_embed = nn.Parameter(torch.zeros(1, self.num_heads, dim))
def forward(self, x, mask=None):
B, N, C = x.shape
# head token
head_pos = self.pos_embed.expand(x.shape[0], -1, -1)
x_ = x.reshape(B, -1, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3)
x_ = x_.mean(dim=2) # now the shape is [B, h, 1, d//h]
x_ = self.ht_proj(x_).reshape(B, -1, self.num_heads, C // self.num_heads)
x_ = self.act(self.ht_norm(x_)).flatten(2)
x_ = x_ + head_pos
x = torch.cat([x, x_], dim=1)
# normal mhsa
qkv = self.qkv(x).reshape(B, N + self.num_heads, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
q, k, v = qkv[0], qkv[1], qkv[2]
attn = (q @ k.transpose(-2, -1)) * self.scale
attn = attn.softmax(dim=-1)
# attn = self.attn_drop(attn) (commented out in tcm_add conformer.py)
x = (attn @ v).transpose(1, 2).reshape(B, N + self.num_heads, C)
x = self.proj(x)
# merge head tokens into cls token
cls, patch, ht = torch.split(x, [1, N - 1, self.num_heads], dim=1)
cls = cls + torch.mean(ht, dim=1, keepdim=True) + torch.mean(patch, dim=1, keepdim=True)
x = torch.cat([cls, patch], dim=1)
x = self.proj_drop(x)
return x, attn
class FeedForward(nn.Module):
def __init__(self, dim, mult=4, dropout=0.):
super().__init__()
self.net = nn.Sequential(
nn.Linear(dim, dim * mult),
Swish(),
nn.Dropout(dropout),
nn.Linear(dim * mult, dim),
nn.Dropout(dropout)
)
def forward(self, x):
return self.net(x)
class ConformerConvModule(nn.Module):
def __init__(self, dim, causal=False, expansion_factor=2, kernel_size=31, dropout=0.):
super().__init__()
inner_dim = dim * expansion_factor
padding = calc_same_padding(kernel_size) if not causal else (kernel_size - 1, 0)
self.net = nn.Sequential(
nn.LayerNorm(dim),
Rearrange('b n c -> b c n'),
nn.Conv1d(dim, inner_dim * 2, 1),
GLU(dim=1),
DepthWiseConv1d(inner_dim, inner_dim, kernel_size=kernel_size, padding=padding),
nn.BatchNorm1d(inner_dim) if not causal else nn.Identity(),
Swish(),
nn.Conv1d(inner_dim, dim, 1),
Rearrange('b c n -> b n c'),
nn.Dropout(dropout)
)
def forward(self, x):
return self.net(x)
class ConformerBlock(nn.Module):
def __init__(self, *, dim, dim_head=64, heads=8, ff_mult=4, conv_expansion_factor=2,
conv_kernel_size=31, attn_dropout=0., ff_dropout=0., conv_dropout=0.,
conv_causal=False):
super().__init__()
self.ff1 = FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout)
self.attn = Attention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout)
self.conv = ConformerConvModule(dim=dim, causal=conv_causal, expansion_factor=conv_expansion_factor,
kernel_size=conv_kernel_size, dropout=conv_dropout)
self.ff2 = FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout)
self.attn = PreNorm(dim, self.attn)
self.ff1 = Scale(0.5, PreNorm(dim, self.ff1))
self.ff2 = Scale(0.5, PreNorm(dim, self.ff2))
self.post_norm = nn.LayerNorm(dim)
def forward(self, x, mask=None):
x = self.ff1(x) + x
attn_x, attn_weight = self.attn(x, mask=mask)
x = attn_x + x
x = self.conv(x) + x
x = self.ff2(x) + x
x = self.post_norm(x)
return x, attn_weight
# ============================ model.py ============================
def sinusoidal_embedding(n_channels, dim):
pe = torch.FloatTensor([[p / (10000 ** (2 * (i // 2) / dim)) for i in range(dim)]
for p in range(n_channels)])
pe[:, 0::2] = torch.sin(pe[:, 0::2])
pe[:, 1::2] = torch.cos(pe[:, 1::2])
return pe.unsqueeze(0)
class MyConformer(nn.Module):
def __init__(self, emb_size=128, heads=4, ffmult=4, exp_fac=2, kernel_size=16, n_encoders=1):
super(MyConformer, self).__init__()
self.dim_head = int(emb_size / heads)
self.dim = emb_size
self.heads = heads
self.kernel_size = kernel_size
self.n_encoders = n_encoders
# edit 6: zeros placeholder; the checkpoint stores this (frozen) table and overwrites it
self.positional_emb = nn.Parameter(torch.zeros(1, 10000, emb_size), requires_grad=False)
self.encoder_blocks = _get_clones(ConformerBlock(dim=emb_size, dim_head=self.dim_head, heads=heads,
ff_mult=ffmult, conv_expansion_factor=exp_fac,
conv_kernel_size=kernel_size),
n_encoders)
self.class_token = nn.Parameter(torch.rand(1, emb_size))
self.fc5 = nn.Linear(emb_size, 2)
def forward(self, x): # x shape [bs, tiempo, frecuencia]
x = x + self.positional_emb[:, :x.size(1), :]
# edit 4: == torch.stack([torch.vstack((self.class_token, x[i])) for i in range(len(x))])
x = torch.cat([self.class_token.unsqueeze(0).expand(x.size(0), -1, -1), x], dim=1) # [bs,1+tiempo,emb]
for layer in self.encoder_blocks:
x, _ = layer(x) # [bs,1+tiempo,emb_size]
embedding = x[:, 0, :] # [bs, emb_size]
out = self.fc5(embedding) # [bs,2]
return out
class SSLModel(nn.Module): # W2V
def __init__(self, cp_path=_XLSR_PATH):
super(SSLModel, self).__init__()
import fairseq
from fairseq.dataclass.utils import merge_with_parent
from fairseq.models.wav2vec.wav2vec2 import Wav2Vec2Config, Wav2Vec2Model
state = fairseq.checkpoint_utils.load_checkpoint_to_cpu(cp_path)
cfg = merge_with_parent(Wav2Vec2Config(), state["cfg"]["model"], remove_missing=True)
del state
self.model = Wav2Vec2Model.build_model(cfg, task=None)
self.out_dim = 1024
def extract_feat(self, input_data):
# edit 2: no `self.model.to(...); self.model.train()` here.
if input_data.ndim == 3:
input_tmp = input_data[:, :, 0]
else:
input_tmp = input_data
# [batch, length, dim]
emb = self.model(input_tmp, mask=False, features_only=True)['x']
return emb
class Model(nn.Module):
def __init__(self, emb_size=144, heads=4, kernel_size=31, num_encoders=4):
super().__init__()
self.ssl_model = SSLModel()
self.LL = nn.Linear(1024, emb_size)
self.first_bn = nn.BatchNorm2d(num_features=1)
self.selu = nn.SELU(inplace=True)
self.conformer = MyConformer(emb_size=emb_size, n_encoders=num_encoders,
heads=heads, kernel_size=kernel_size)
def forward(self, x):
x_ssl_feat = self.ssl_model.extract_feat(x.squeeze(-1))
x = self.LL(x_ssl_feat) # (bs,frame_number,feat_out_dim)
x = x.unsqueeze(dim=1) # add channel
x = self.first_bn(x)
x = self.selu(x)
x = x.squeeze(dim=1)
out = self.conformer(x)
return out