Simih/final_yor-audio-lm

YorAudioLM

YorAudioLM is a Yoruba speech model that pairs a frozen Whisper-large-v3 audio encoder with AfriqueQwen3.5-9B-50Langs-Instruct-v1 - an instruction tuned LLM pre-trained on 26 billion tokens across 50 African languages — via a linear projector. The model produces correctly diacritized Yoruba transcriptions and can translate Yoruba speech directly to English.

This is an early checkpoint trained on Yoruba ASR with Stage 1 projector alignment followed by LoRA fine-tuning. A full multilingual version covering Yoruba, Igbo, Hausa,
and Nigerian Pidgin is in progress.

Inference

import torch                                                                                                                                                                                              
import librosa                                                                                                                                                                                            
from transformers import AutoProcessor, AutoModelForCausalLM
                                                                                                                                                                                                          
processor = AutoProcessor.from_pretrained("Simih/final_yor-audio-lm", trust_remote_code=True)                                                                                                           
model = AutoModelForCausalLM.from_pretrained(                                                                                                                                                             
    "Simih/final_yor-audio-lm", torch_dtype=torch.bfloat16, device_map="cuda", trust_remote_code=True                                                                                                     
)                                                                                                                                                                                                         
model.eval()                                                                                                                                                                                              
                                                                                                                                                                                                        
audio, _ = librosa.load("your_audio.wav", sr=16000, mono=True)                                                                                                                                            
audio_block = processor.audio_bos_token + processor.audio_token * 750 + processor.audio_eos_token                                                                                                       
                                                                                                                                                                                                          
# transcription                                                                                                                                                                                         
task = "Transcribe the audio."                                                                                                                                                                            
                                                                                                                                                                                                          
# English translation                                                                                                                                                                                     
# task = "Translate this audio to English."                                                                                                                                                               
                                                                                                                                                                                                          
# transcribe then translate (chain-of-thought)                                                                                                                                                            
# task = "Transcribe then translate this audio to English."                                                                                                                                               
                                                                                                                                                                                                          
text = f"<|im_start|>user\n{audio_block}{task}<|im_end|>\n<|im_start|>assistant\n"                                                                                                                        
inputs = processor(text=text, audio=audio, return_tensors="pt").to("cuda")                                                                                                                                
im_end_id = processor.tokenizer.convert_tokens_to_ids("<|im_end|>")                                                                                                                                       
                                                                                                                                                                                                          
with torch.no_grad():                                                                                                                                                                                     
    output_ids = model.generate(                                                                                                                                                                          
        **inputs,                                                                                                                                                                                         
        max_new_tokens=256,                                                                                                                                                                               
        do_sample=False,                                                                                                                                                                                  
        eos_token_id=[processor.tokenizer.eos_token_id, im_end_id],                                                                                                                                     
    )                                                                                                                                                                                                     
                                                                                                                                                                                                          
print(processor.batch_decode(output_ids, skip_special_tokens=True)[0])