almubdimutaikhan's picture
Qolda-AVL-5B-OptimizedTokenizer: weights, audio adapter, loader, card
ac1a5a4 verified
Raw History Blame Contribute Delete
11.3 kB
"""Part 3: our audio-vision-language model = MA_a1.0_pin_VL (frozen) + audio encoder (frozen) + connector (trained).
Only the connector has trainable weights, unless an audio-only LoRA is attached (attach_lora;
the base LLM weights stay unchanged, and with the adapter disabled the model is exactly the original). Audio enters exactly like images do in Qwen3.5: the prompt holds
<|audio_start|> + N x <|audio_pad|> + <|audio_end|>, and the N <|audio_pad|> embeddings are replaced by the N
connector outputs. Qwen3.5 has no DeepStack, so this is input-only. Text and vision paths are untouched.
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file
AUDIO_START, AUDIO_END, AUDIO_PAD = 248070, 248071, 248076
# step 2 LoRA targets in the Qwen3.5 language model: full attention (8 layers), linear attention (24), MLP (32)
LORA_TARGETS = (r".*layers\.\d+\.(self_attn\.(q_proj|k_proj|v_proj|o_proj)"
r"|linear_attn\.(in_proj_qkv|in_proj_z|out_proj)|mlp\.(gate_proj|up_proj|down_proj))$")
IM_END, EOT = 248046, 248044
class Connector(nn.Module):
"""avg-pool over time (x pool, ceil) -> LayerNorm -> Linear -> GELU -> Linear. Same MLP as Qolda-AVL's
audio_projection, so for Whisper (d_in 1280) it can start from AVL's trained weights."""
def __init__(self, d_in, d_llm=2560, pool=1, d_extra=0):
super().__init__()
self.pool, self.d_in, self.d_extra = pool, d_in, d_extra
self.norm = nn.LayerNorm(d_in)
self.fc1 = nn.Linear(d_in, d_llm)
self.fc2 = nn.Linear(d_llm, d_llm)
if d_extra: # extra encoder layers (gender fix): own LayerNorm + zero-initialised projection added to fc1's
self.norm_x = nn.LayerNorm(d_extra) # output, so a fresh extra branch leaves the output unchanged
self.fc1_x = nn.Linear(d_extra, d_llm, bias=False)
nn.init.zeros_(self.fc1_x.weight)
def load_compat(self, sd):
"""Load a connector state dict; a connector trained without extra layers is accepted (extra branch stays
zero, so the output equals the old connector's)."""
sd = {k: v.float() for k, v in sd.items()}
missing, unexpected = self.load_state_dict(sd, strict=False)
assert not unexpected, unexpected
assert all(k.startswith(("norm_x.", "fc1_x.")) for k in missing), missing
return missing
def forward(self, frames): # list of [T_i, d_in] -> [sum ceil(T_i/pool), d_llm]
out = []
for x in frames:
x = x.float()
if self.pool > 1:
x = F.avg_pool1d(x.T[None], self.pool, self.pool, ceil_mode=True)[0].T
out.append(x)
x = torch.cat(out)
if self.d_extra:
assert x.shape[1] == self.d_in + self.d_extra, (x.shape, self.d_in, self.d_extra)
xm, xe = x[:, :self.d_in], x[:, self.d_in:]
return self.fc2(F.gelu(self.fc1(self.norm(xm)) + self.fc1_x(self.norm_x(xe))))
return self.fc2(F.gelu(self.fc1(self.norm(x))))
class OurAVL(nn.Module):
def __init__(self, vl_path, encoder, init_connector=False, grad_ckpt=True, attn="sdpa"):
super().__init__()
from transformers import Qwen3_5ForConditionalGeneration
self.vl = Qwen3_5ForConditionalGeneration.from_pretrained(vl_path, dtype=torch.bfloat16, attn_implementation=attn)
self.vl.requires_grad_(False)
self.lm = self.vl.model.language_model
self.lm_head = self.vl.lm_head
self.encoder = encoder
self.connector = Connector(encoder.d_out, self.lm.config.hidden_size, encoder.pool,
getattr(encoder, "d_extra", 0))
if init_connector:
assert encoder.connector_init, f"{encoder.name} has no connector_init"
self.connector.load_state_dict({k: v.float() for k, v in load_file(encoder.connector_init).items()})
if grad_ckpt:
self.lm.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
def train(self, mode=True): # only the connector (and checkpointing in lm) see train mode
super().train(mode)
self.encoder.eval()
self.vl.eval()
self.lm.train(mode) # needed for gradient checkpointing; Qwen3.5 has no dropout
return self
def embed(self, input_ids, audio):
emb = self.lm.embed_tokens(input_ids)
if audio is not None:
with torch.no_grad():
frames = self.encoder(audio)
a = self.connector(frames).to(emb.dtype)
mask = input_ids == AUDIO_PAD
assert int(mask.sum()) == a.shape[0], f"{int(mask.sum())} <|audio_pad|> vs {a.shape[0]} audio vectors"
emb = emb.masked_scatter(mask[..., None], a)
return emb
def forward(self, input_ids, attention_mask, labels, audio=None, chunk=4096):
emb = self.embed(input_ids, audio)
h = self.lm(inputs_embeds=emb, attention_mask=attention_mask, use_cache=False).last_hidden_state
# next-token loss on answer tokens only, computed in chunks (vocab is 264k, full logits would be huge)
tgt = labels[:, 1:]
sel = tgt != -100
hs, ys = h[:, :-1][sel], tgt[sel]
loss_sum = h.new_zeros((), dtype=torch.float32)
for i in range(0, hs.shape[0], chunk):
logits = self.lm_head(hs[i:i + chunk]).float()
loss_sum = loss_sum + F.cross_entropy(logits, ys[i:i + chunk], reduction="sum")
return loss_sum, int(ys.numel())
@torch.no_grad()
def generate(self, input_ids, audio=None, max_new_tokens=256, eos=(IM_END, EOT)):
"""Greedy decoding for one example (batch 1), used for WER evaluation."""
emb = self.embed(input_ids, audio)
out = self.lm(inputs_embeds=emb, use_cache=True)
past, new = out.past_key_values, []
nxt = self.lm_head(out.last_hidden_state[:, -1]).argmax(-1)
for _ in range(max_new_tokens):
t = int(nxt)
if t in eos:
break
new.append(t)
out = self.lm(inputs_embeds=self.lm.embed_tokens(nxt[:, None]), past_key_values=past, use_cache=True)
past = out.past_key_values
nxt = self.lm_head(out.last_hidden_state[:, -1]).argmax(-1)
return new
@staticmethod
def _pick(logits, temperature=0.0, top_p=1.0, top_k=0):
"""next token: greedy if temperature <= 0, else temperature + top-k + top-p sampling"""
if temperature <= 0:
return logits.argmax(-1)
logits = logits.float() / temperature
if top_k:
kth = torch.topk(logits, top_k, dim=-1).values[:, -1:]
logits = logits.masked_fill(logits < kth, float("-inf"))
if top_p < 1.0:
srt, idx = torch.sort(logits, descending=True, dim=-1)
pr = torch.softmax(srt, -1)
srt = srt.masked_fill(pr.cumsum(-1) - pr > top_p, float("-inf"))
logits = torch.full_like(logits, float("-inf")).scatter(-1, idx, srt)
return torch.multinomial(torch.softmax(logits, -1), 1)[:, 0]
@torch.no_grad()
def generate_batch(self, prompts, audio, max_new_tokens=256, eos=(IM_END, EOT), pad_id=EOT,
temperature=0.0, top_p=1.0, top_k=0):
"""Greedy (default) or sampled decoding for a batch (left-padded). prompts: list of token-id lists; audio: encoder.collate(...) in
the same order. Returns a list of generated token-id lists. Relative positions (RoPE) are unaffected by the
left padding, and padded positions are masked in both attention types."""
dev = self.lm_head.weight.device
T = max(len(p) for p in prompts)
ids = torch.full((len(prompts), T), pad_id, dtype=torch.long, device=dev)
att = torch.zeros((len(prompts), T), dtype=torch.long, device=dev)
for i, p in enumerate(prompts):
ids[i, T - len(p):] = torch.tensor(p, device=dev); att[i, T - len(p):] = 1
emb = self.embed(ids, audio)
out = self.lm(inputs_embeds=emb, attention_mask=att, use_cache=True)
past = out.past_key_values
nxt = self._pick(self.lm_head(out.last_hidden_state[:, -1]), temperature, top_p, top_k)
done = torch.zeros(len(prompts), dtype=torch.bool, device=dev)
gen = [[] for _ in prompts]
for _ in range(max_new_tokens):
for i, t in enumerate(nxt.tolist()):
if not done[i]:
if t in eos:
done[i] = True
else:
gen[i].append(t)
if bool(done.all()):
break
nxt = torch.where(done, torch.full_like(nxt, pad_id), nxt)
att = torch.cat([att, torch.ones((len(prompts), 1), dtype=att.dtype, device=dev)], 1)
out = self.lm(inputs_embeds=self.lm.embed_tokens(nxt[:, None]), attention_mask=att, past_key_values=past,
use_cache=True)
past = out.past_key_values
nxt = self._pick(self.lm_head(out.last_hidden_state[:, -1]), temperature, top_p, top_k)
return gen
# ---------------------------------------------------------------- step 2: audio-only LoRA on the LLM
def attach_lora(self, r=16, alpha=32):
"""Inject a LoRA adapter ("audio") into the language model; base weights stay frozen. Adapter weights fp32."""
from peft import LoraConfig, inject_adapter_in_model
cfg = LoraConfig(r=r, lora_alpha=alpha, lora_dropout=0.0, bias="none", target_modules=LORA_TARGETS)
inject_adapter_in_model(cfg, self.lm, adapter_name="audio")
for n, p in self.lm.named_parameters():
if "lora_" in n:
p.data = p.data.float()
p.requires_grad_(True)
self.lora_cfg = {"r": r, "alpha": alpha}
return [p for n, p in self.lm.named_parameters() if "lora_" in n]
def set_lora(self, enabled):
"""enabled=False -> every LoRA layer is bypassed: outputs are exactly the original model's."""
from peft.tuners.tuners_utils import BaseTunerLayer
for m in self.lm.modules():
if isinstance(m, BaseTunerLayer):
m.enable_adapters(enabled)
def lora_state(self):
return {n: p.detach().cpu() for n, p in self.lm.named_parameters() if "lora_" in n}
def load_lora(self, path):
"""attach (r/alpha from the file's metadata) and load an adapter saved by train_avl.py"""
from safetensors import safe_open
with safe_open(path, "pt") as f:
meta = f.metadata() or {}
if not hasattr(self, "lora_cfg"):
self.attach_lora(int(meta.get("r", 16)), int(meta.get("alpha", 32)))
sd = load_file(path)
own = dict(self.lm.named_parameters())
missing = [k for k in own if "lora_" in k and k not in sd]
assert not missing and all(k in own for k in sd), f"LoRA mismatch: {missing[:3]}"
with torch.no_grad():
for k, v in sd.items():
own[k].copy_(v.to(own[k].dtype))
def connector_state(self):
return {k: v.detach().cpu() for k, v in self.connector.state_dict().items()}