import gradio as gr import spaces import torch import torchaudio from PIL import Image from app.services.image_caption import ImageCaptioner from app.services.audio_caption import AudioCaptioner # Initialize models image_captioner = ImageCaptioner() audio_captioner = AudioCaptioner() def load_any_audio(audio_path, target_sr=32000): """Robust audio loader supporting .wav, .mp3, .ogg, and .flac""" try: wav, sr = torchaudio.load(audio_path) except Exception: import soundfile as sf data, sr = sf.read(audio_path, dtype="float32") wav = torch.from_numpy(data) if wav.dim() == 1: wav = wav.unsqueeze(0) elif wav.dim() == 2: wav = wav.t() # Stereo to mono if wav.dim() == 2 and wav.shape[0] > 1: wav = wav.mean(dim=0, keepdim=True) elif wav.dim() == 1: wav = wav.unsqueeze(0) # Resample to model's canonical rate if sr != target_sr: wav = torchaudio.functional.resample(wav, sr, target_sr) return wav @spaces.GPU def generate_image_caption(image_path): if image_path is None: return "Please upload an image." try: image = Image.open(image_path).convert("RGB") device = next(image_captioner.model.parameters()).device inputs = image_captioner.processor(images=image, return_tensors="pt") inputs = {k: v.to(device) for k, v in inputs.items()} with torch.no_grad(): output = image_captioner.model.generate(**inputs, max_new_tokens=50) caption = image_captioner.processor.decode(output[0], skip_special_tokens=True) return caption except Exception as e: return f"Error processing image: {str(e)}" @spaces.GPU def generate_audio_caption(audio_path): if audio_path is None: return "Please upload an audio file." try: # Load any audio format (.wav, .mp3, .ogg) target_sr = getattr(audio_captioner.model.config, "sample_rate", 32000) wav = load_any_audio(audio_path, target_sr=target_sr) # Match tensor device to model device dynamically (CUDA or CPU) device = next(audio_captioner.model.parameters()).device wav = wav.to(device) audio_length = [wav.size(1)] with torch.no_grad(): output = audio_captioner.model(audio=wav, audio_length=audio_length) caption = audio_captioner.tokenizer.decode(output[0], skip_special_tokens=True) return caption except Exception as e: return f"Error processing audio: {str(e)}" # Gradio Interface with gr.Blocks(title="Multimodal Caption Generator", theme=gr.themes.Soft()) as demo: gr.Markdown("Multimodal Caption Generator") with gr.Tab("📷 Image Captioning"): with gr.Row(): img_input = gr.Image(type="filepath", label="Upload Image (JPG, PNG, WebP)") img_output = gr.Textbox(label="Generated Caption", lines=3) img_button = gr.Button("Generate Image Caption", variant="primary") img_button.click(generate_image_caption, inputs=img_input, outputs=img_output) with gr.Tab("🎵 Audio Captioning"): with gr.Row(): aud_input = gr.Audio(type="filepath", label="Upload Audio (.wav, .mp3, .ogg)") aud_output = gr.Textbox(label="Generated Caption", lines=3) aud_button = gr.Button("Generate Audio Caption", variant="primary") aud_button.click(generate_audio_caption, inputs=aud_input, outputs=aud_output) demo.launch()