"""Part 3: our audio-vision-language model = MA_a1.0_pin_VL (frozen) + audio encoder (frozen) + connector (trained). Only the connector has trainable weights, unless an audio-only LoRA is attached (attach_lora; the base LLM weights stay unchanged, and with the adapter disabled the model is exactly the original). Audio enters exactly like images do in Qwen3.5: the prompt holds <|audio_start|> + N x <|audio_pad|> + <|audio_end|>, and the N <|audio_pad|> embeddings are replaced by the N connector outputs. Qwen3.5 has no DeepStack, so this is input-only. Text and vision paths are untouched. """ import math import torch import torch.nn as nn import torch.nn.functional as F from safetensors.torch import load_file AUDIO_START, AUDIO_END, AUDIO_PAD = 248070, 248071, 248076 # step 2 LoRA targets in the Qwen3.5 language model: full attention (8 layers), linear attention (24), MLP (32) LORA_TARGETS = (r".*layers\.\d+\.(self_attn\.(q_proj|k_proj|v_proj|o_proj)" r"|linear_attn\.(in_proj_qkv|in_proj_z|out_proj)|mlp\.(gate_proj|up_proj|down_proj))$") IM_END, EOT = 248046, 248044 class Connector(nn.Module): """avg-pool over time (x pool, ceil) -> LayerNorm -> Linear -> GELU -> Linear. Same MLP as Qolda-AVL's audio_projection, so for Whisper (d_in 1280) it can start from AVL's trained weights.""" def __init__(self, d_in, d_llm=2560, pool=1, d_extra=0): super().__init__() self.pool, self.d_in, self.d_extra = pool, d_in, d_extra self.norm = nn.LayerNorm(d_in) self.fc1 = nn.Linear(d_in, d_llm) self.fc2 = nn.Linear(d_llm, d_llm) if d_extra: # extra encoder layers (gender fix): own LayerNorm + zero-initialised projection added to fc1's self.norm_x = nn.LayerNorm(d_extra) # output, so a fresh extra branch leaves the output unchanged self.fc1_x = nn.Linear(d_extra, d_llm, bias=False) nn.init.zeros_(self.fc1_x.weight) def load_compat(self, sd): """Load a connector state dict; a connector trained without extra layers is accepted (extra branch stays zero, so the output equals the old connector's).""" sd = {k: v.float() for k, v in sd.items()} missing, unexpected = self.load_state_dict(sd, strict=False) assert not unexpected, unexpected assert all(k.startswith(("norm_x.", "fc1_x.")) for k in missing), missing return missing def forward(self, frames): # list of [T_i, d_in] -> [sum ceil(T_i/pool), d_llm] out = [] for x in frames: x = x.float() if self.pool > 1: x = F.avg_pool1d(x.T[None], self.pool, self.pool, ceil_mode=True)[0].T out.append(x) x = torch.cat(out) if self.d_extra: assert x.shape[1] == self.d_in + self.d_extra, (x.shape, self.d_in, self.d_extra) xm, xe = x[:, :self.d_in], x[:, self.d_in:] return self.fc2(F.gelu(self.fc1(self.norm(xm)) + self.fc1_x(self.norm_x(xe)))) return self.fc2(F.gelu(self.fc1(self.norm(x)))) class OurAVL(nn.Module): def __init__(self, vl_path, encoder, init_connector=False, grad_ckpt=True, attn="sdpa"): super().__init__() from transformers import Qwen3_5ForConditionalGeneration self.vl = Qwen3_5ForConditionalGeneration.from_pretrained(vl_path, dtype=torch.bfloat16, attn_implementation=attn) self.vl.requires_grad_(False) self.lm = self.vl.model.language_model self.lm_head = self.vl.lm_head self.encoder = encoder self.connector = Connector(encoder.d_out, self.lm.config.hidden_size, encoder.pool, getattr(encoder, "d_extra", 0)) if init_connector: assert encoder.connector_init, f"{encoder.name} has no connector_init" self.connector.load_state_dict({k: v.float() for k, v in load_file(encoder.connector_init).items()}) if grad_ckpt: self.lm.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) def train(self, mode=True): # only the connector (and checkpointing in lm) see train mode super().train(mode) self.encoder.eval() self.vl.eval() self.lm.train(mode) # needed for gradient checkpointing; Qwen3.5 has no dropout return self def embed(self, input_ids, audio): emb = self.lm.embed_tokens(input_ids) if audio is not None: with torch.no_grad(): frames = self.encoder(audio) a = self.connector(frames).to(emb.dtype) mask = input_ids == AUDIO_PAD assert int(mask.sum()) == a.shape[0], f"{int(mask.sum())} <|audio_pad|> vs {a.shape[0]} audio vectors" emb = emb.masked_scatter(mask[..., None], a) return emb def forward(self, input_ids, attention_mask, labels, audio=None, chunk=4096): emb = self.embed(input_ids, audio) h = self.lm(inputs_embeds=emb, attention_mask=attention_mask, use_cache=False).last_hidden_state # next-token loss on answer tokens only, computed in chunks (vocab is 264k, full logits would be huge) tgt = labels[:, 1:] sel = tgt != -100 hs, ys = h[:, :-1][sel], tgt[sel] loss_sum = h.new_zeros((), dtype=torch.float32) for i in range(0, hs.shape[0], chunk): logits = self.lm_head(hs[i:i + chunk]).float() loss_sum = loss_sum + F.cross_entropy(logits, ys[i:i + chunk], reduction="sum") return loss_sum, int(ys.numel()) @torch.no_grad() def generate(self, input_ids, audio=None, max_new_tokens=256, eos=(IM_END, EOT)): """Greedy decoding for one example (batch 1), used for WER evaluation.""" emb = self.embed(input_ids, audio) out = self.lm(inputs_embeds=emb, use_cache=True) past, new = out.past_key_values, [] nxt = self.lm_head(out.last_hidden_state[:, -1]).argmax(-1) for _ in range(max_new_tokens): t = int(nxt) if t in eos: break new.append(t) out = self.lm(inputs_embeds=self.lm.embed_tokens(nxt[:, None]), past_key_values=past, use_cache=True) past = out.past_key_values nxt = self.lm_head(out.last_hidden_state[:, -1]).argmax(-1) return new @staticmethod def _pick(logits, temperature=0.0, top_p=1.0, top_k=0): """next token: greedy if temperature <= 0, else temperature + top-k + top-p sampling""" if temperature <= 0: return logits.argmax(-1) logits = logits.float() / temperature if top_k: kth = torch.topk(logits, top_k, dim=-1).values[:, -1:] logits = logits.masked_fill(logits < kth, float("-inf")) if top_p < 1.0: srt, idx = torch.sort(logits, descending=True, dim=-1) pr = torch.softmax(srt, -1) srt = srt.masked_fill(pr.cumsum(-1) - pr > top_p, float("-inf")) logits = torch.full_like(logits, float("-inf")).scatter(-1, idx, srt) return torch.multinomial(torch.softmax(logits, -1), 1)[:, 0] @torch.no_grad() def generate_batch(self, prompts, audio, max_new_tokens=256, eos=(IM_END, EOT), pad_id=EOT, temperature=0.0, top_p=1.0, top_k=0): """Greedy (default) or sampled decoding for a batch (left-padded). prompts: list of token-id lists; audio: encoder.collate(...) in the same order. Returns a list of generated token-id lists. Relative positions (RoPE) are unaffected by the left padding, and padded positions are masked in both attention types.""" dev = self.lm_head.weight.device T = max(len(p) for p in prompts) ids = torch.full((len(prompts), T), pad_id, dtype=torch.long, device=dev) att = torch.zeros((len(prompts), T), dtype=torch.long, device=dev) for i, p in enumerate(prompts): ids[i, T - len(p):] = torch.tensor(p, device=dev); att[i, T - len(p):] = 1 emb = self.embed(ids, audio) out = self.lm(inputs_embeds=emb, attention_mask=att, use_cache=True) past = out.past_key_values nxt = self._pick(self.lm_head(out.last_hidden_state[:, -1]), temperature, top_p, top_k) done = torch.zeros(len(prompts), dtype=torch.bool, device=dev) gen = [[] for _ in prompts] for _ in range(max_new_tokens): for i, t in enumerate(nxt.tolist()): if not done[i]: if t in eos: done[i] = True else: gen[i].append(t) if bool(done.all()): break nxt = torch.where(done, torch.full_like(nxt, pad_id), nxt) att = torch.cat([att, torch.ones((len(prompts), 1), dtype=att.dtype, device=dev)], 1) out = self.lm(inputs_embeds=self.lm.embed_tokens(nxt[:, None]), attention_mask=att, past_key_values=past, use_cache=True) past = out.past_key_values nxt = self._pick(self.lm_head(out.last_hidden_state[:, -1]), temperature, top_p, top_k) return gen # ---------------------------------------------------------------- step 2: audio-only LoRA on the LLM def attach_lora(self, r=16, alpha=32): """Inject a LoRA adapter ("audio") into the language model; base weights stay frozen. Adapter weights fp32.""" from peft import LoraConfig, inject_adapter_in_model cfg = LoraConfig(r=r, lora_alpha=alpha, lora_dropout=0.0, bias="none", target_modules=LORA_TARGETS) inject_adapter_in_model(cfg, self.lm, adapter_name="audio") for n, p in self.lm.named_parameters(): if "lora_" in n: p.data = p.data.float() p.requires_grad_(True) self.lora_cfg = {"r": r, "alpha": alpha} return [p for n, p in self.lm.named_parameters() if "lora_" in n] def set_lora(self, enabled): """enabled=False -> every LoRA layer is bypassed: outputs are exactly the original model's.""" from peft.tuners.tuners_utils import BaseTunerLayer for m in self.lm.modules(): if isinstance(m, BaseTunerLayer): m.enable_adapters(enabled) def lora_state(self): return {n: p.detach().cpu() for n, p in self.lm.named_parameters() if "lora_" in n} def load_lora(self, path): """attach (r/alpha from the file's metadata) and load an adapter saved by train_avl.py""" from safetensors import safe_open with safe_open(path, "pt") as f: meta = f.metadata() or {} if not hasattr(self, "lora_cfg"): self.attach_lora(int(meta.get("r", 16)), int(meta.get("alpha", 32))) sd = load_file(path) own = dict(self.lm.named_parameters()) missing = [k for k in own if "lora_" in k and k not in sd] assert not missing and all(k in own for k in sd), f"LoRA mismatch: {missing[:3]}" with torch.no_grad(): for k, v in sd.items(): own[k].copy_(v.to(own[k].dtype)) def connector_state(self): return {k: v.detach().cpu() for k, v in self.connector.state_dict().items()}