"""XLSR-Conformer + TCM (Temporal-Channel Modeling) network, ported from https://github.com/ductuantruong/tcm_add (MIT) @ 726387eb731223a21d551476349f8f8f680a3375: conformer.py (ConformerBlock + Head-Token Attention, verbatim below) model.py (sinusoidal_embedding, MyConformer, SSLModel, Model) Truong et al., "Temporal-Channel Modeling in Multi-head Self-Attention for Synthetic Speech Detection", Interspeech 2024. Same network as benchmarks/DPDA-GA-TCM/_net.py (dpda_ga re-used tcm_add's code; the only source differences are listed in edits 5, 7). Hyper-parameters = main.py defaults: emb_size 144, heads 4, kernel_size 31, num_encoders 4 (ff_mult 4, exp_fac 2); verified against the released checkpoint shapes (strict load). Edits (inference-only, numerically identical; parameter/module names unchanged so the released state_dict loads strictly): 1. SSLModel builds the XLS-R 300M wav2vec2 architecture from the base xlsr2_300m.pt model cfg merged onto the installed fairseq Wav2Vec2Config (merge_with_parent; cross-version cfg-field fix, as W2V2-AASIST/_net.py), instead of load_model_ensemble_and_task on a hard-coded path. The base weights are not loaded -- the fine-tuned checkpoint overwrites every weight. 2. SSLModel.extract_feat: the `self.model.train()` call on a device/dtype mismatch is removed (it would re-enable dropout at inference); the wrapper moves the net to the device and calls .eval() itself. 3. Model.__init__ takes the hyper-parameters as keyword args (default = main.py) instead of an argparse namespace; `device` args dropped (unused at inference). 4. MyConformer.forward: the class-token prepend `torch.stack([torch.vstack((cls, x[i])) for i in range(len(x))])` is written as `torch.cat([cls.expand(B, 1, E), x], 1)` (same tensor; the python loop over the batch would freeze the batch size into a traced ONNX graph). 5. Model.forward / MyConformer.forward return only the (B, 2) logits (source returns (out, list_of_attention_maps); main.py uses only out). 6. MyConformer.positional_emb is initialised to zeros instead of the slow python-list sinusoid build: it is a frozen nn.Parameter stored in (and loaded from) the checkpoint. 7. (not an edit) Attention.forward keeps the source's commented-out attn_drop. """ import os import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from einops.layers.torch import Rearrange from torch.nn.modules.transformer import _get_clones _XLSR_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "xlsr2_300m.pt") WINDOW = 66800 # source data_utils.Dataset_eval: self.cut = 66800 (fixed-length eval) def pad(x: np.ndarray, max_len: int = WINDOW) -> np.ndarray: """source utils.pad: first max_len samples; shorter clips repeat-tiled.""" x = np.asarray(x, dtype=np.float32).reshape(-1) x_len = x.shape[0] if x_len >= max_len: return x[:max_len] num_repeats = int(max_len / x_len) + 1 return np.tile(x, (1, num_repeats))[:, :max_len][0] # ============================ conformer.py ============================ def exists(val): return val is not None def default(val, d): return val if exists(val) else d def calc_same_padding(kernel_size): pad = kernel_size // 2 return (pad, pad - (kernel_size + 1) % 2) class Swish(nn.Module): def forward(self, x): return x * x.sigmoid() class GLU(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, x): out, gate = x.chunk(2, dim=self.dim) return out * gate.sigmoid() class DepthWiseConv1d(nn.Module): def __init__(self, chan_in, chan_out, kernel_size, padding): super().__init__() self.padding = padding self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, groups=chan_in) def forward(self, x): x = F.pad(x, self.padding) return self.conv(x) class Scale(nn.Module): def __init__(self, scale, fn): super().__init__() self.fn = fn self.scale = scale def forward(self, x, **kwargs): return self.fn(x, **kwargs) * self.scale class PreNorm(nn.Module): def __init__(self, dim, fn): super().__init__() self.fn = fn self.norm = nn.LayerNorm(dim) def forward(self, x, **kwargs): x = self.norm(x) return self.fn(x, **kwargs) class Attention(nn.Module): # Head Token attention: https://arxiv.org/pdf/2210.05958.pdf def __init__(self, dim, heads=8, dim_head=64, qkv_bias=False, dropout=0., proj_drop=0.): super().__init__() self.num_heads = heads inner_dim = dim_head * heads self.scale = dim_head ** -0.5 self.qkv = nn.Linear(dim, inner_dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(dropout) self.proj = nn.Linear(inner_dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.act = nn.GELU() self.ht_proj = nn.Linear(dim_head, dim, bias=True) self.ht_norm = nn.LayerNorm(dim_head) self.pos_embed = nn.Parameter(torch.zeros(1, self.num_heads, dim)) def forward(self, x, mask=None): B, N, C = x.shape # head token head_pos = self.pos_embed.expand(x.shape[0], -1, -1) x_ = x.reshape(B, -1, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) x_ = x_.mean(dim=2) # now the shape is [B, h, 1, d//h] x_ = self.ht_proj(x_).reshape(B, -1, self.num_heads, C // self.num_heads) x_ = self.act(self.ht_norm(x_)).flatten(2) x_ = x_ + head_pos x = torch.cat([x, x_], dim=1) # normal mhsa qkv = self.qkv(x).reshape(B, N + self.num_heads, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv[0], qkv[1], qkv[2] attn = (q @ k.transpose(-2, -1)) * self.scale attn = attn.softmax(dim=-1) # attn = self.attn_drop(attn) (commented out in tcm_add conformer.py) x = (attn @ v).transpose(1, 2).reshape(B, N + self.num_heads, C) x = self.proj(x) # merge head tokens into cls token cls, patch, ht = torch.split(x, [1, N - 1, self.num_heads], dim=1) cls = cls + torch.mean(ht, dim=1, keepdim=True) + torch.mean(patch, dim=1, keepdim=True) x = torch.cat([cls, patch], dim=1) x = self.proj_drop(x) return x, attn class FeedForward(nn.Module): def __init__(self, dim, mult=4, dropout=0.): super().__init__() self.net = nn.Sequential( nn.Linear(dim, dim * mult), Swish(), nn.Dropout(dropout), nn.Linear(dim * mult, dim), nn.Dropout(dropout) ) def forward(self, x): return self.net(x) class ConformerConvModule(nn.Module): def __init__(self, dim, causal=False, expansion_factor=2, kernel_size=31, dropout=0.): super().__init__() inner_dim = dim * expansion_factor padding = calc_same_padding(kernel_size) if not causal else (kernel_size - 1, 0) self.net = nn.Sequential( nn.LayerNorm(dim), Rearrange('b n c -> b c n'), nn.Conv1d(dim, inner_dim * 2, 1), GLU(dim=1), DepthWiseConv1d(inner_dim, inner_dim, kernel_size=kernel_size, padding=padding), nn.BatchNorm1d(inner_dim) if not causal else nn.Identity(), Swish(), nn.Conv1d(inner_dim, dim, 1), Rearrange('b c n -> b n c'), nn.Dropout(dropout) ) def forward(self, x): return self.net(x) class ConformerBlock(nn.Module): def __init__(self, *, dim, dim_head=64, heads=8, ff_mult=4, conv_expansion_factor=2, conv_kernel_size=31, attn_dropout=0., ff_dropout=0., conv_dropout=0., conv_causal=False): super().__init__() self.ff1 = FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout) self.attn = Attention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout) self.conv = ConformerConvModule(dim=dim, causal=conv_causal, expansion_factor=conv_expansion_factor, kernel_size=conv_kernel_size, dropout=conv_dropout) self.ff2 = FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout) self.attn = PreNorm(dim, self.attn) self.ff1 = Scale(0.5, PreNorm(dim, self.ff1)) self.ff2 = Scale(0.5, PreNorm(dim, self.ff2)) self.post_norm = nn.LayerNorm(dim) def forward(self, x, mask=None): x = self.ff1(x) + x attn_x, attn_weight = self.attn(x, mask=mask) x = attn_x + x x = self.conv(x) + x x = self.ff2(x) + x x = self.post_norm(x) return x, attn_weight # ============================ model.py ============================ def sinusoidal_embedding(n_channels, dim): pe = torch.FloatTensor([[p / (10000 ** (2 * (i // 2) / dim)) for i in range(dim)] for p in range(n_channels)]) pe[:, 0::2] = torch.sin(pe[:, 0::2]) pe[:, 1::2] = torch.cos(pe[:, 1::2]) return pe.unsqueeze(0) class MyConformer(nn.Module): def __init__(self, emb_size=128, heads=4, ffmult=4, exp_fac=2, kernel_size=16, n_encoders=1): super(MyConformer, self).__init__() self.dim_head = int(emb_size / heads) self.dim = emb_size self.heads = heads self.kernel_size = kernel_size self.n_encoders = n_encoders # edit 6: zeros placeholder; the checkpoint stores this (frozen) table and overwrites it self.positional_emb = nn.Parameter(torch.zeros(1, 10000, emb_size), requires_grad=False) self.encoder_blocks = _get_clones(ConformerBlock(dim=emb_size, dim_head=self.dim_head, heads=heads, ff_mult=ffmult, conv_expansion_factor=exp_fac, conv_kernel_size=kernel_size), n_encoders) self.class_token = nn.Parameter(torch.rand(1, emb_size)) self.fc5 = nn.Linear(emb_size, 2) def forward(self, x): # x shape [bs, tiempo, frecuencia] x = x + self.positional_emb[:, :x.size(1), :] # edit 4: == torch.stack([torch.vstack((self.class_token, x[i])) for i in range(len(x))]) x = torch.cat([self.class_token.unsqueeze(0).expand(x.size(0), -1, -1), x], dim=1) # [bs,1+tiempo,emb] for layer in self.encoder_blocks: x, _ = layer(x) # [bs,1+tiempo,emb_size] embedding = x[:, 0, :] # [bs, emb_size] out = self.fc5(embedding) # [bs,2] return out class SSLModel(nn.Module): # W2V def __init__(self, cp_path=_XLSR_PATH): super(SSLModel, self).__init__() import fairseq from fairseq.dataclass.utils import merge_with_parent from fairseq.models.wav2vec.wav2vec2 import Wav2Vec2Config, Wav2Vec2Model state = fairseq.checkpoint_utils.load_checkpoint_to_cpu(cp_path) cfg = merge_with_parent(Wav2Vec2Config(), state["cfg"]["model"], remove_missing=True) del state self.model = Wav2Vec2Model.build_model(cfg, task=None) self.out_dim = 1024 def extract_feat(self, input_data): # edit 2: no `self.model.to(...); self.model.train()` here. if input_data.ndim == 3: input_tmp = input_data[:, :, 0] else: input_tmp = input_data # [batch, length, dim] emb = self.model(input_tmp, mask=False, features_only=True)['x'] return emb class Model(nn.Module): def __init__(self, emb_size=144, heads=4, kernel_size=31, num_encoders=4): super().__init__() self.ssl_model = SSLModel() self.LL = nn.Linear(1024, emb_size) self.first_bn = nn.BatchNorm2d(num_features=1) self.selu = nn.SELU(inplace=True) self.conformer = MyConformer(emb_size=emb_size, n_encoders=num_encoders, heads=heads, kernel_size=kernel_size) def forward(self, x): x_ssl_feat = self.ssl_model.extract_feat(x.squeeze(-1)) x = self.LL(x_ssl_feat) # (bs,frame_number,feat_out_dim) x = x.unsqueeze(dim=1) # add channel x = self.first_bn(x) x = self.selu(x) x = x.squeeze(dim=1) out = self.conformer(x) return out