Download app.py from AmanPanwar/multimodal-caption-generator: direct link, hf CLI and curl.
- Browser
- Download file 3.51 kB
-
https://huggingface.co/spaces/AmanPanwar/multimodal-caption-generator/resolve/main/app.py
- Command line
-
hf download hf://spaces/AmanPanwar/multimodal-caption-generator/app.py
-
curl -L -o app.py https://huggingface.co/spaces/AmanPanwar/multimodal-caption-generator/resolve/main/app.py
3.51 kB
| import gradio as gr | |
| import spaces | |
| import torch | |
| import torchaudio | |
| from PIL import Image | |
| from app.services.image_caption import ImageCaptioner | |
| from app.services.audio_caption import AudioCaptioner | |
| # Initialize models | |
| image_captioner = ImageCaptioner() | |
| audio_captioner = AudioCaptioner() | |
| def load_any_audio(audio_path, target_sr=32000): | |
| """Robust audio loader supporting .wav, .mp3, .ogg, and .flac""" | |
| try: | |
| wav, sr = torchaudio.load(audio_path) | |
| except Exception: | |
| import soundfile as sf | |
| data, sr = sf.read(audio_path, dtype="float32") | |
| wav = torch.from_numpy(data) | |
| if wav.dim() == 1: | |
| wav = wav.unsqueeze(0) | |
| elif wav.dim() == 2: | |
| wav = wav.t() | |
| # Stereo to mono | |
| if wav.dim() == 2 and wav.shape[0] > 1: | |
| wav = wav.mean(dim=0, keepdim=True) | |
| elif wav.dim() == 1: | |
| wav = wav.unsqueeze(0) | |
| # Resample to model's canonical rate | |
| if sr != target_sr: | |
| wav = torchaudio.functional.resample(wav, sr, target_sr) | |
| return wav | |
| def generate_image_caption(image_path): | |
| if image_path is None: | |
| return "Please upload an image." | |
| try: | |
| image = Image.open(image_path).convert("RGB") | |
| device = next(image_captioner.model.parameters()).device | |
| inputs = image_captioner.processor(images=image, return_tensors="pt") | |
| inputs = {k: v.to(device) for k, v in inputs.items()} | |
| with torch.no_grad(): | |
| output = image_captioner.model.generate(**inputs, max_new_tokens=50) | |
| caption = image_captioner.processor.decode(output[0], skip_special_tokens=True) | |
| return caption | |
| except Exception as e: | |
| return f"Error processing image: {str(e)}" | |
| def generate_audio_caption(audio_path): | |
| if audio_path is None: | |
| return "Please upload an audio file." | |
| try: | |
| # Load any audio format (.wav, .mp3, .ogg) | |
| target_sr = getattr(audio_captioner.model.config, "sample_rate", 32000) | |
| wav = load_any_audio(audio_path, target_sr=target_sr) | |
| # Match tensor device to model device dynamically (CUDA or CPU) | |
| device = next(audio_captioner.model.parameters()).device | |
| wav = wav.to(device) | |
| audio_length = [wav.size(1)] | |
| with torch.no_grad(): | |
| output = audio_captioner.model(audio=wav, audio_length=audio_length) | |
| caption = audio_captioner.tokenizer.decode(output[0], skip_special_tokens=True) | |
| return caption | |
| except Exception as e: | |
| return f"Error processing audio: {str(e)}" | |
| # Gradio Interface | |
| with gr.Blocks(title="Multimodal Caption Generator", theme=gr.themes.Soft()) as demo: | |
| gr.Markdown("Multimodal Caption Generator") | |
| with gr.Tab("📷 Image Captioning"): | |
| with gr.Row(): | |
| img_input = gr.Image(type="filepath", label="Upload Image (JPG, PNG, WebP)") | |
| img_output = gr.Textbox(label="Generated Caption", lines=3) | |
| img_button = gr.Button("Generate Image Caption", variant="primary") | |
| img_button.click(generate_image_caption, inputs=img_input, outputs=img_output) | |
| with gr.Tab("🎵 Audio Captioning"): | |
| with gr.Row(): | |
| aud_input = gr.Audio(type="filepath", label="Upload Audio (.wav, .mp3, .ogg)") | |
| aud_output = gr.Textbox(label="Generated Caption", lines=3) | |
| aud_button = gr.Button("Generate Audio Caption", variant="primary") | |
| aud_button.click(generate_audio_caption, inputs=aud_input, outputs=aud_output) | |
| demo.launch() | |