AmanPanwar's picture
Update app.py
e9e40f9 verified
Raw History Blame Contribute Delete
3.51 kB
import gradio as gr
import spaces
import torch
import torchaudio
from PIL import Image
from app.services.image_caption import ImageCaptioner
from app.services.audio_caption import AudioCaptioner
# Initialize models
image_captioner = ImageCaptioner()
audio_captioner = AudioCaptioner()
def load_any_audio(audio_path, target_sr=32000):
"""Robust audio loader supporting .wav, .mp3, .ogg, and .flac"""
try:
wav, sr = torchaudio.load(audio_path)
except Exception:
import soundfile as sf
data, sr = sf.read(audio_path, dtype="float32")
wav = torch.from_numpy(data)
if wav.dim() == 1:
wav = wav.unsqueeze(0)
elif wav.dim() == 2:
wav = wav.t()
# Stereo to mono
if wav.dim() == 2 and wav.shape[0] > 1:
wav = wav.mean(dim=0, keepdim=True)
elif wav.dim() == 1:
wav = wav.unsqueeze(0)
# Resample to model's canonical rate
if sr != target_sr:
wav = torchaudio.functional.resample(wav, sr, target_sr)
return wav
@spaces.GPU
def generate_image_caption(image_path):
if image_path is None:
return "Please upload an image."
try:
image = Image.open(image_path).convert("RGB")
device = next(image_captioner.model.parameters()).device
inputs = image_captioner.processor(images=image, return_tensors="pt")
inputs = {k: v.to(device) for k, v in inputs.items()}
with torch.no_grad():
output = image_captioner.model.generate(**inputs, max_new_tokens=50)
caption = image_captioner.processor.decode(output[0], skip_special_tokens=True)
return caption
except Exception as e:
return f"Error processing image: {str(e)}"
@spaces.GPU
def generate_audio_caption(audio_path):
if audio_path is None:
return "Please upload an audio file."
try:
# Load any audio format (.wav, .mp3, .ogg)
target_sr = getattr(audio_captioner.model.config, "sample_rate", 32000)
wav = load_any_audio(audio_path, target_sr=target_sr)
# Match tensor device to model device dynamically (CUDA or CPU)
device = next(audio_captioner.model.parameters()).device
wav = wav.to(device)
audio_length = [wav.size(1)]
with torch.no_grad():
output = audio_captioner.model(audio=wav, audio_length=audio_length)
caption = audio_captioner.tokenizer.decode(output[0], skip_special_tokens=True)
return caption
except Exception as e:
return f"Error processing audio: {str(e)}"
# Gradio Interface
with gr.Blocks(title="Multimodal Caption Generator", theme=gr.themes.Soft()) as demo:
gr.Markdown("Multimodal Caption Generator")
with gr.Tab("📷 Image Captioning"):
with gr.Row():
img_input = gr.Image(type="filepath", label="Upload Image (JPG, PNG, WebP)")
img_output = gr.Textbox(label="Generated Caption", lines=3)
img_button = gr.Button("Generate Image Caption", variant="primary")
img_button.click(generate_image_caption, inputs=img_input, outputs=img_output)
with gr.Tab("🎵 Audio Captioning"):
with gr.Row():
aud_input = gr.Audio(type="filepath", label="Upload Audio (.wav, .mp3, .ogg)")
aud_output = gr.Textbox(label="Generated Caption", lines=3)
aud_button = gr.Button("Generate Audio Caption", variant="primary")
aud_button.click(generate_audio_caption, inputs=aud_input, outputs=aud_output)
demo.launch()