Download _net.py from SpeechAntiSpoofingBenchmarks/XLSR-Conformer-TCM-DF: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/SpeechAntiSpoofingBenchmarks/XLSR-Conformer-TCM-DF/resolve/main/_net.py
- Command line
-
hf download hf://SpeechAntiSpoofingBenchmarks/XLSR-Conformer-TCM-DF/_net.py
-
curl -L -o _net.py https://huggingface.co/SpeechAntiSpoofingBenchmarks/XLSR-Conformer-TCM-DF/resolve/main/_net.py
12.5 kB
| """XLSR-Conformer + TCM (Temporal-Channel Modeling) network, ported from | |
| https://github.com/ductuantruong/tcm_add (MIT) @ 726387eb731223a21d551476349f8f8f680a3375: | |
| conformer.py (ConformerBlock + Head-Token Attention, verbatim below) | |
| model.py (sinusoidal_embedding, MyConformer, SSLModel, Model) | |
| Truong et al., "Temporal-Channel Modeling in Multi-head Self-Attention for Synthetic | |
| Speech Detection", Interspeech 2024. Same network as benchmarks/DPDA-GA-TCM/_net.py | |
| (dpda_ga re-used tcm_add's code; the only source differences are listed in edits 5, 7). | |
| Hyper-parameters = main.py defaults: emb_size 144, heads 4, kernel_size 31, | |
| num_encoders 4 (ff_mult 4, exp_fac 2); verified against the released checkpoint | |
| shapes (strict load). | |
| Edits (inference-only, numerically identical; parameter/module names unchanged so | |
| the released state_dict loads strictly): | |
| 1. SSLModel builds the XLS-R 300M wav2vec2 architecture from the base | |
| xlsr2_300m.pt model cfg merged onto the installed fairseq Wav2Vec2Config | |
| (merge_with_parent; cross-version cfg-field fix, as W2V2-AASIST/_net.py), | |
| instead of load_model_ensemble_and_task on a hard-coded path. The base | |
| weights are not loaded -- the fine-tuned checkpoint overwrites every weight. | |
| 2. SSLModel.extract_feat: the `self.model.train()` call on a device/dtype | |
| mismatch is removed (it would re-enable dropout at inference); the wrapper | |
| moves the net to the device and calls .eval() itself. | |
| 3. Model.__init__ takes the hyper-parameters as keyword args (default = main.py) | |
| instead of an argparse namespace; `device` args dropped (unused at inference). | |
| 4. MyConformer.forward: the class-token prepend | |
| `torch.stack([torch.vstack((cls, x[i])) for i in range(len(x))])` is written as | |
| `torch.cat([cls.expand(B, 1, E), x], 1)` (same tensor; the python loop over the | |
| batch would freeze the batch size into a traced ONNX graph). | |
| 5. Model.forward / MyConformer.forward return only the (B, 2) logits (source returns | |
| (out, list_of_attention_maps); main.py uses only out). | |
| 6. MyConformer.positional_emb is initialised to zeros instead of the slow python-list | |
| sinusoid build: it is a frozen nn.Parameter stored in (and loaded from) the checkpoint. | |
| 7. (not an edit) Attention.forward keeps the source's commented-out attn_drop. | |
| """ | |
| import os | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from einops.layers.torch import Rearrange | |
| from torch.nn.modules.transformer import _get_clones | |
| _XLSR_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "xlsr2_300m.pt") | |
| WINDOW = 66800 # source data_utils.Dataset_eval: self.cut = 66800 (fixed-length eval) | |
| def pad(x: np.ndarray, max_len: int = WINDOW) -> np.ndarray: | |
| """source utils.pad: first max_len samples; shorter clips repeat-tiled.""" | |
| x = np.asarray(x, dtype=np.float32).reshape(-1) | |
| x_len = x.shape[0] | |
| if x_len >= max_len: | |
| return x[:max_len] | |
| num_repeats = int(max_len / x_len) + 1 | |
| return np.tile(x, (1, num_repeats))[:, :max_len][0] | |
| # ============================ conformer.py ============================ | |
| def exists(val): | |
| return val is not None | |
| def default(val, d): | |
| return val if exists(val) else d | |
| def calc_same_padding(kernel_size): | |
| pad = kernel_size // 2 | |
| return (pad, pad - (kernel_size + 1) % 2) | |
| class Swish(nn.Module): | |
| def forward(self, x): | |
| return x * x.sigmoid() | |
| class GLU(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.dim = dim | |
| def forward(self, x): | |
| out, gate = x.chunk(2, dim=self.dim) | |
| return out * gate.sigmoid() | |
| class DepthWiseConv1d(nn.Module): | |
| def __init__(self, chan_in, chan_out, kernel_size, padding): | |
| super().__init__() | |
| self.padding = padding | |
| self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, groups=chan_in) | |
| def forward(self, x): | |
| x = F.pad(x, self.padding) | |
| return self.conv(x) | |
| class Scale(nn.Module): | |
| def __init__(self, scale, fn): | |
| super().__init__() | |
| self.fn = fn | |
| self.scale = scale | |
| def forward(self, x, **kwargs): | |
| return self.fn(x, **kwargs) * self.scale | |
| class PreNorm(nn.Module): | |
| def __init__(self, dim, fn): | |
| super().__init__() | |
| self.fn = fn | |
| self.norm = nn.LayerNorm(dim) | |
| def forward(self, x, **kwargs): | |
| x = self.norm(x) | |
| return self.fn(x, **kwargs) | |
| class Attention(nn.Module): | |
| # Head Token attention: https://arxiv.org/pdf/2210.05958.pdf | |
| def __init__(self, dim, heads=8, dim_head=64, qkv_bias=False, dropout=0., proj_drop=0.): | |
| super().__init__() | |
| self.num_heads = heads | |
| inner_dim = dim_head * heads | |
| self.scale = dim_head ** -0.5 | |
| self.qkv = nn.Linear(dim, inner_dim * 3, bias=qkv_bias) | |
| self.attn_drop = nn.Dropout(dropout) | |
| self.proj = nn.Linear(inner_dim, dim) | |
| self.proj_drop = nn.Dropout(proj_drop) | |
| self.act = nn.GELU() | |
| self.ht_proj = nn.Linear(dim_head, dim, bias=True) | |
| self.ht_norm = nn.LayerNorm(dim_head) | |
| self.pos_embed = nn.Parameter(torch.zeros(1, self.num_heads, dim)) | |
| def forward(self, x, mask=None): | |
| B, N, C = x.shape | |
| # head token | |
| head_pos = self.pos_embed.expand(x.shape[0], -1, -1) | |
| x_ = x.reshape(B, -1, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) | |
| x_ = x_.mean(dim=2) # now the shape is [B, h, 1, d//h] | |
| x_ = self.ht_proj(x_).reshape(B, -1, self.num_heads, C // self.num_heads) | |
| x_ = self.act(self.ht_norm(x_)).flatten(2) | |
| x_ = x_ + head_pos | |
| x = torch.cat([x, x_], dim=1) | |
| # normal mhsa | |
| qkv = self.qkv(x).reshape(B, N + self.num_heads, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) | |
| q, k, v = qkv[0], qkv[1], qkv[2] | |
| attn = (q @ k.transpose(-2, -1)) * self.scale | |
| attn = attn.softmax(dim=-1) | |
| # attn = self.attn_drop(attn) (commented out in tcm_add conformer.py) | |
| x = (attn @ v).transpose(1, 2).reshape(B, N + self.num_heads, C) | |
| x = self.proj(x) | |
| # merge head tokens into cls token | |
| cls, patch, ht = torch.split(x, [1, N - 1, self.num_heads], dim=1) | |
| cls = cls + torch.mean(ht, dim=1, keepdim=True) + torch.mean(patch, dim=1, keepdim=True) | |
| x = torch.cat([cls, patch], dim=1) | |
| x = self.proj_drop(x) | |
| return x, attn | |
| class FeedForward(nn.Module): | |
| def __init__(self, dim, mult=4, dropout=0.): | |
| super().__init__() | |
| self.net = nn.Sequential( | |
| nn.Linear(dim, dim * mult), | |
| Swish(), | |
| nn.Dropout(dropout), | |
| nn.Linear(dim * mult, dim), | |
| nn.Dropout(dropout) | |
| ) | |
| def forward(self, x): | |
| return self.net(x) | |
| class ConformerConvModule(nn.Module): | |
| def __init__(self, dim, causal=False, expansion_factor=2, kernel_size=31, dropout=0.): | |
| super().__init__() | |
| inner_dim = dim * expansion_factor | |
| padding = calc_same_padding(kernel_size) if not causal else (kernel_size - 1, 0) | |
| self.net = nn.Sequential( | |
| nn.LayerNorm(dim), | |
| Rearrange('b n c -> b c n'), | |
| nn.Conv1d(dim, inner_dim * 2, 1), | |
| GLU(dim=1), | |
| DepthWiseConv1d(inner_dim, inner_dim, kernel_size=kernel_size, padding=padding), | |
| nn.BatchNorm1d(inner_dim) if not causal else nn.Identity(), | |
| Swish(), | |
| nn.Conv1d(inner_dim, dim, 1), | |
| Rearrange('b c n -> b n c'), | |
| nn.Dropout(dropout) | |
| ) | |
| def forward(self, x): | |
| return self.net(x) | |
| class ConformerBlock(nn.Module): | |
| def __init__(self, *, dim, dim_head=64, heads=8, ff_mult=4, conv_expansion_factor=2, | |
| conv_kernel_size=31, attn_dropout=0., ff_dropout=0., conv_dropout=0., | |
| conv_causal=False): | |
| super().__init__() | |
| self.ff1 = FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout) | |
| self.attn = Attention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout) | |
| self.conv = ConformerConvModule(dim=dim, causal=conv_causal, expansion_factor=conv_expansion_factor, | |
| kernel_size=conv_kernel_size, dropout=conv_dropout) | |
| self.ff2 = FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout) | |
| self.attn = PreNorm(dim, self.attn) | |
| self.ff1 = Scale(0.5, PreNorm(dim, self.ff1)) | |
| self.ff2 = Scale(0.5, PreNorm(dim, self.ff2)) | |
| self.post_norm = nn.LayerNorm(dim) | |
| def forward(self, x, mask=None): | |
| x = self.ff1(x) + x | |
| attn_x, attn_weight = self.attn(x, mask=mask) | |
| x = attn_x + x | |
| x = self.conv(x) + x | |
| x = self.ff2(x) + x | |
| x = self.post_norm(x) | |
| return x, attn_weight | |
| # ============================ model.py ============================ | |
| def sinusoidal_embedding(n_channels, dim): | |
| pe = torch.FloatTensor([[p / (10000 ** (2 * (i // 2) / dim)) for i in range(dim)] | |
| for p in range(n_channels)]) | |
| pe[:, 0::2] = torch.sin(pe[:, 0::2]) | |
| pe[:, 1::2] = torch.cos(pe[:, 1::2]) | |
| return pe.unsqueeze(0) | |
| class MyConformer(nn.Module): | |
| def __init__(self, emb_size=128, heads=4, ffmult=4, exp_fac=2, kernel_size=16, n_encoders=1): | |
| super(MyConformer, self).__init__() | |
| self.dim_head = int(emb_size / heads) | |
| self.dim = emb_size | |
| self.heads = heads | |
| self.kernel_size = kernel_size | |
| self.n_encoders = n_encoders | |
| # edit 6: zeros placeholder; the checkpoint stores this (frozen) table and overwrites it | |
| self.positional_emb = nn.Parameter(torch.zeros(1, 10000, emb_size), requires_grad=False) | |
| self.encoder_blocks = _get_clones(ConformerBlock(dim=emb_size, dim_head=self.dim_head, heads=heads, | |
| ff_mult=ffmult, conv_expansion_factor=exp_fac, | |
| conv_kernel_size=kernel_size), | |
| n_encoders) | |
| self.class_token = nn.Parameter(torch.rand(1, emb_size)) | |
| self.fc5 = nn.Linear(emb_size, 2) | |
| def forward(self, x): # x shape [bs, tiempo, frecuencia] | |
| x = x + self.positional_emb[:, :x.size(1), :] | |
| # edit 4: == torch.stack([torch.vstack((self.class_token, x[i])) for i in range(len(x))]) | |
| x = torch.cat([self.class_token.unsqueeze(0).expand(x.size(0), -1, -1), x], dim=1) # [bs,1+tiempo,emb] | |
| for layer in self.encoder_blocks: | |
| x, _ = layer(x) # [bs,1+tiempo,emb_size] | |
| embedding = x[:, 0, :] # [bs, emb_size] | |
| out = self.fc5(embedding) # [bs,2] | |
| return out | |
| class SSLModel(nn.Module): # W2V | |
| def __init__(self, cp_path=_XLSR_PATH): | |
| super(SSLModel, self).__init__() | |
| import fairseq | |
| from fairseq.dataclass.utils import merge_with_parent | |
| from fairseq.models.wav2vec.wav2vec2 import Wav2Vec2Config, Wav2Vec2Model | |
| state = fairseq.checkpoint_utils.load_checkpoint_to_cpu(cp_path) | |
| cfg = merge_with_parent(Wav2Vec2Config(), state["cfg"]["model"], remove_missing=True) | |
| del state | |
| self.model = Wav2Vec2Model.build_model(cfg, task=None) | |
| self.out_dim = 1024 | |
| def extract_feat(self, input_data): | |
| # edit 2: no `self.model.to(...); self.model.train()` here. | |
| if input_data.ndim == 3: | |
| input_tmp = input_data[:, :, 0] | |
| else: | |
| input_tmp = input_data | |
| # [batch, length, dim] | |
| emb = self.model(input_tmp, mask=False, features_only=True)['x'] | |
| return emb | |
| class Model(nn.Module): | |
| def __init__(self, emb_size=144, heads=4, kernel_size=31, num_encoders=4): | |
| super().__init__() | |
| self.ssl_model = SSLModel() | |
| self.LL = nn.Linear(1024, emb_size) | |
| self.first_bn = nn.BatchNorm2d(num_features=1) | |
| self.selu = nn.SELU(inplace=True) | |
| self.conformer = MyConformer(emb_size=emb_size, n_encoders=num_encoders, | |
| heads=heads, kernel_size=kernel_size) | |
| def forward(self, x): | |
| x_ssl_feat = self.ssl_model.extract_feat(x.squeeze(-1)) | |
| x = self.LL(x_ssl_feat) # (bs,frame_number,feat_out_dim) | |
| x = x.unsqueeze(dim=1) # add channel | |
| x = self.first_bn(x) | |
| x = self.selu(x) | |
| x = x.squeeze(dim=1) | |
| out = self.conformer(x) | |
| return out | |