Download avl_code/model_avl.py from issai/Qolda-AVL-5B-OptimizedTokenizer: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/issai/Qolda-AVL-5B-OptimizedTokenizer/resolve/main/avl_code/model_avl.py
- Command line
-
hf download hf://issai/Qolda-AVL-5B-OptimizedTokenizer/avl_code/model_avl.py
-
curl -L -o model_avl.py https://huggingface.co/issai/Qolda-AVL-5B-OptimizedTokenizer/resolve/main/avl_code/model_avl.py
11.3 kB
| """Part 3: our audio-vision-language model = MA_a1.0_pin_VL (frozen) + audio encoder (frozen) + connector (trained). | |
| Only the connector has trainable weights, unless an audio-only LoRA is attached (attach_lora; | |
| the base LLM weights stay unchanged, and with the adapter disabled the model is exactly the original). Audio enters exactly like images do in Qwen3.5: the prompt holds | |
| <|audio_start|> + N x <|audio_pad|> + <|audio_end|>, and the N <|audio_pad|> embeddings are replaced by the N | |
| connector outputs. Qwen3.5 has no DeepStack, so this is input-only. Text and vision paths are untouched. | |
| """ | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| AUDIO_START, AUDIO_END, AUDIO_PAD = 248070, 248071, 248076 | |
| # step 2 LoRA targets in the Qwen3.5 language model: full attention (8 layers), linear attention (24), MLP (32) | |
| LORA_TARGETS = (r".*layers\.\d+\.(self_attn\.(q_proj|k_proj|v_proj|o_proj)" | |
| r"|linear_attn\.(in_proj_qkv|in_proj_z|out_proj)|mlp\.(gate_proj|up_proj|down_proj))$") | |
| IM_END, EOT = 248046, 248044 | |
| class Connector(nn.Module): | |
| """avg-pool over time (x pool, ceil) -> LayerNorm -> Linear -> GELU -> Linear. Same MLP as Qolda-AVL's | |
| audio_projection, so for Whisper (d_in 1280) it can start from AVL's trained weights.""" | |
| def __init__(self, d_in, d_llm=2560, pool=1, d_extra=0): | |
| super().__init__() | |
| self.pool, self.d_in, self.d_extra = pool, d_in, d_extra | |
| self.norm = nn.LayerNorm(d_in) | |
| self.fc1 = nn.Linear(d_in, d_llm) | |
| self.fc2 = nn.Linear(d_llm, d_llm) | |
| if d_extra: # extra encoder layers (gender fix): own LayerNorm + zero-initialised projection added to fc1's | |
| self.norm_x = nn.LayerNorm(d_extra) # output, so a fresh extra branch leaves the output unchanged | |
| self.fc1_x = nn.Linear(d_extra, d_llm, bias=False) | |
| nn.init.zeros_(self.fc1_x.weight) | |
| def load_compat(self, sd): | |
| """Load a connector state dict; a connector trained without extra layers is accepted (extra branch stays | |
| zero, so the output equals the old connector's).""" | |
| sd = {k: v.float() for k, v in sd.items()} | |
| missing, unexpected = self.load_state_dict(sd, strict=False) | |
| assert not unexpected, unexpected | |
| assert all(k.startswith(("norm_x.", "fc1_x.")) for k in missing), missing | |
| return missing | |
| def forward(self, frames): # list of [T_i, d_in] -> [sum ceil(T_i/pool), d_llm] | |
| out = [] | |
| for x in frames: | |
| x = x.float() | |
| if self.pool > 1: | |
| x = F.avg_pool1d(x.T[None], self.pool, self.pool, ceil_mode=True)[0].T | |
| out.append(x) | |
| x = torch.cat(out) | |
| if self.d_extra: | |
| assert x.shape[1] == self.d_in + self.d_extra, (x.shape, self.d_in, self.d_extra) | |
| xm, xe = x[:, :self.d_in], x[:, self.d_in:] | |
| return self.fc2(F.gelu(self.fc1(self.norm(xm)) + self.fc1_x(self.norm_x(xe)))) | |
| return self.fc2(F.gelu(self.fc1(self.norm(x)))) | |
| class OurAVL(nn.Module): | |
| def __init__(self, vl_path, encoder, init_connector=False, grad_ckpt=True, attn="sdpa"): | |
| super().__init__() | |
| from transformers import Qwen3_5ForConditionalGeneration | |
| self.vl = Qwen3_5ForConditionalGeneration.from_pretrained(vl_path, dtype=torch.bfloat16, attn_implementation=attn) | |
| self.vl.requires_grad_(False) | |
| self.lm = self.vl.model.language_model | |
| self.lm_head = self.vl.lm_head | |
| self.encoder = encoder | |
| self.connector = Connector(encoder.d_out, self.lm.config.hidden_size, encoder.pool, | |
| getattr(encoder, "d_extra", 0)) | |
| if init_connector: | |
| assert encoder.connector_init, f"{encoder.name} has no connector_init" | |
| self.connector.load_state_dict({k: v.float() for k, v in load_file(encoder.connector_init).items()}) | |
| if grad_ckpt: | |
| self.lm.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) | |
| def train(self, mode=True): # only the connector (and checkpointing in lm) see train mode | |
| super().train(mode) | |
| self.encoder.eval() | |
| self.vl.eval() | |
| self.lm.train(mode) # needed for gradient checkpointing; Qwen3.5 has no dropout | |
| return self | |
| def embed(self, input_ids, audio): | |
| emb = self.lm.embed_tokens(input_ids) | |
| if audio is not None: | |
| with torch.no_grad(): | |
| frames = self.encoder(audio) | |
| a = self.connector(frames).to(emb.dtype) | |
| mask = input_ids == AUDIO_PAD | |
| assert int(mask.sum()) == a.shape[0], f"{int(mask.sum())} <|audio_pad|> vs {a.shape[0]} audio vectors" | |
| emb = emb.masked_scatter(mask[..., None], a) | |
| return emb | |
| def forward(self, input_ids, attention_mask, labels, audio=None, chunk=4096): | |
| emb = self.embed(input_ids, audio) | |
| h = self.lm(inputs_embeds=emb, attention_mask=attention_mask, use_cache=False).last_hidden_state | |
| # next-token loss on answer tokens only, computed in chunks (vocab is 264k, full logits would be huge) | |
| tgt = labels[:, 1:] | |
| sel = tgt != -100 | |
| hs, ys = h[:, :-1][sel], tgt[sel] | |
| loss_sum = h.new_zeros((), dtype=torch.float32) | |
| for i in range(0, hs.shape[0], chunk): | |
| logits = self.lm_head(hs[i:i + chunk]).float() | |
| loss_sum = loss_sum + F.cross_entropy(logits, ys[i:i + chunk], reduction="sum") | |
| return loss_sum, int(ys.numel()) | |
| def generate(self, input_ids, audio=None, max_new_tokens=256, eos=(IM_END, EOT)): | |
| """Greedy decoding for one example (batch 1), used for WER evaluation.""" | |
| emb = self.embed(input_ids, audio) | |
| out = self.lm(inputs_embeds=emb, use_cache=True) | |
| past, new = out.past_key_values, [] | |
| nxt = self.lm_head(out.last_hidden_state[:, -1]).argmax(-1) | |
| for _ in range(max_new_tokens): | |
| t = int(nxt) | |
| if t in eos: | |
| break | |
| new.append(t) | |
| out = self.lm(inputs_embeds=self.lm.embed_tokens(nxt[:, None]), past_key_values=past, use_cache=True) | |
| past = out.past_key_values | |
| nxt = self.lm_head(out.last_hidden_state[:, -1]).argmax(-1) | |
| return new | |
| def _pick(logits, temperature=0.0, top_p=1.0, top_k=0): | |
| """next token: greedy if temperature <= 0, else temperature + top-k + top-p sampling""" | |
| if temperature <= 0: | |
| return logits.argmax(-1) | |
| logits = logits.float() / temperature | |
| if top_k: | |
| kth = torch.topk(logits, top_k, dim=-1).values[:, -1:] | |
| logits = logits.masked_fill(logits < kth, float("-inf")) | |
| if top_p < 1.0: | |
| srt, idx = torch.sort(logits, descending=True, dim=-1) | |
| pr = torch.softmax(srt, -1) | |
| srt = srt.masked_fill(pr.cumsum(-1) - pr > top_p, float("-inf")) | |
| logits = torch.full_like(logits, float("-inf")).scatter(-1, idx, srt) | |
| return torch.multinomial(torch.softmax(logits, -1), 1)[:, 0] | |
| def generate_batch(self, prompts, audio, max_new_tokens=256, eos=(IM_END, EOT), pad_id=EOT, | |
| temperature=0.0, top_p=1.0, top_k=0): | |
| """Greedy (default) or sampled decoding for a batch (left-padded). prompts: list of token-id lists; audio: encoder.collate(...) in | |
| the same order. Returns a list of generated token-id lists. Relative positions (RoPE) are unaffected by the | |
| left padding, and padded positions are masked in both attention types.""" | |
| dev = self.lm_head.weight.device | |
| T = max(len(p) for p in prompts) | |
| ids = torch.full((len(prompts), T), pad_id, dtype=torch.long, device=dev) | |
| att = torch.zeros((len(prompts), T), dtype=torch.long, device=dev) | |
| for i, p in enumerate(prompts): | |
| ids[i, T - len(p):] = torch.tensor(p, device=dev); att[i, T - len(p):] = 1 | |
| emb = self.embed(ids, audio) | |
| out = self.lm(inputs_embeds=emb, attention_mask=att, use_cache=True) | |
| past = out.past_key_values | |
| nxt = self._pick(self.lm_head(out.last_hidden_state[:, -1]), temperature, top_p, top_k) | |
| done = torch.zeros(len(prompts), dtype=torch.bool, device=dev) | |
| gen = [[] for _ in prompts] | |
| for _ in range(max_new_tokens): | |
| for i, t in enumerate(nxt.tolist()): | |
| if not done[i]: | |
| if t in eos: | |
| done[i] = True | |
| else: | |
| gen[i].append(t) | |
| if bool(done.all()): | |
| break | |
| nxt = torch.where(done, torch.full_like(nxt, pad_id), nxt) | |
| att = torch.cat([att, torch.ones((len(prompts), 1), dtype=att.dtype, device=dev)], 1) | |
| out = self.lm(inputs_embeds=self.lm.embed_tokens(nxt[:, None]), attention_mask=att, past_key_values=past, | |
| use_cache=True) | |
| past = out.past_key_values | |
| nxt = self._pick(self.lm_head(out.last_hidden_state[:, -1]), temperature, top_p, top_k) | |
| return gen | |
| # ---------------------------------------------------------------- step 2: audio-only LoRA on the LLM | |
| def attach_lora(self, r=16, alpha=32): | |
| """Inject a LoRA adapter ("audio") into the language model; base weights stay frozen. Adapter weights fp32.""" | |
| from peft import LoraConfig, inject_adapter_in_model | |
| cfg = LoraConfig(r=r, lora_alpha=alpha, lora_dropout=0.0, bias="none", target_modules=LORA_TARGETS) | |
| inject_adapter_in_model(cfg, self.lm, adapter_name="audio") | |
| for n, p in self.lm.named_parameters(): | |
| if "lora_" in n: | |
| p.data = p.data.float() | |
| p.requires_grad_(True) | |
| self.lora_cfg = {"r": r, "alpha": alpha} | |
| return [p for n, p in self.lm.named_parameters() if "lora_" in n] | |
| def set_lora(self, enabled): | |
| """enabled=False -> every LoRA layer is bypassed: outputs are exactly the original model's.""" | |
| from peft.tuners.tuners_utils import BaseTunerLayer | |
| for m in self.lm.modules(): | |
| if isinstance(m, BaseTunerLayer): | |
| m.enable_adapters(enabled) | |
| def lora_state(self): | |
| return {n: p.detach().cpu() for n, p in self.lm.named_parameters() if "lora_" in n} | |
| def load_lora(self, path): | |
| """attach (r/alpha from the file's metadata) and load an adapter saved by train_avl.py""" | |
| from safetensors import safe_open | |
| with safe_open(path, "pt") as f: | |
| meta = f.metadata() or {} | |
| if not hasattr(self, "lora_cfg"): | |
| self.attach_lora(int(meta.get("r", 16)), int(meta.get("alpha", 32))) | |
| sd = load_file(path) | |
| own = dict(self.lm.named_parameters()) | |
| missing = [k for k in own if "lora_" in k and k not in sd] | |
| assert not missing and all(k in own for k in sd), f"LoRA mismatch: {missing[:3]}" | |
| with torch.no_grad(): | |
| for k, v in sd.items(): | |
| own[k].copy_(v.to(own[k].dtype)) | |
| def connector_state(self): | |
| return {k: v.detach().cpu() for k, v in self.connector.state_dict().items()} | |