Delete app.py
Browse files
app.py
DELETED
|
@@ -1,92 +0,0 @@
|
|
| 1 |
-
import gradio as gr
|
| 2 |
-
from gtts import gTTS
|
| 3 |
-
import os
|
| 4 |
-
from PIL import Image
|
| 5 |
-
from transformers import pipeline, AutoModelForSeq2SeqLM, AutoTokenizer, BlipProcessor, BlipForConditionalGeneration
|
| 6 |
-
|
| 7 |
-
# Define translation models per language pair
|
| 8 |
-
translation_models = {
|
| 9 |
-
"French": "Helsinki-NLP/opus-mt-en-fr",
|
| 10 |
-
"Spanish": "Helsinki-NLP/opus-mt-en-es",
|
| 11 |
-
"Mandarin": "Helsinki-NLP/opus-mt-en-zh",
|
| 12 |
-
"Vietnamese": "Helsinki-NLP/opus-mt-en-vi",
|
| 13 |
-
"Tagalog": "Helsinki-NLP/opus-mt-en-tl"
|
| 14 |
-
}
|
| 15 |
-
|
| 16 |
-
# Directory to store local models
|
| 17 |
-
local_model_dir = "./models"
|
| 18 |
-
os.makedirs(local_model_dir, exist_ok=True)
|
| 19 |
-
|
| 20 |
-
# Load translation models and tokenizers locally
|
| 21 |
-
translation_pipelines = {}
|
| 22 |
-
for lang, model_name in translation_models.items():
|
| 23 |
-
local_path = os.path.join(local_model_dir, model_name.replace('/', '_'))
|
| 24 |
-
|
| 25 |
-
if not os.path.exists(local_path):
|
| 26 |
-
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
|
| 27 |
-
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 28 |
-
model.save_pretrained(local_path)
|
| 29 |
-
tokenizer.save_pretrained(local_path)
|
| 30 |
-
else:
|
| 31 |
-
model = AutoModelForSeq2SeqLM.from_pretrained(local_path)
|
| 32 |
-
tokenizer = AutoTokenizer.from_pretrained(local_path)
|
| 33 |
-
|
| 34 |
-
translation_pipelines[lang] = pipeline("translation", model=model, tokenizer=tokenizer)
|
| 35 |
-
|
| 36 |
-
# Load image captioning model
|
| 37 |
-
caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
|
| 38 |
-
caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
|
| 39 |
-
|
| 40 |
-
def generate_caption(image):
|
| 41 |
-
inputs = caption_processor(image, return_tensors="pt")
|
| 42 |
-
output = caption_model.generate(**inputs)
|
| 43 |
-
caption = caption_processor.decode(output[0], skip_special_tokens=True)
|
| 44 |
-
return caption
|
| 45 |
-
|
| 46 |
-
# Function to translate text using locally stored models
|
| 47 |
-
def translate_text(sentence, target_lang):
|
| 48 |
-
if target_lang in translation_pipelines:
|
| 49 |
-
model = translation_pipelines[target_lang]
|
| 50 |
-
translation = model(sentence, max_length=100, truncation=True)
|
| 51 |
-
return translation[0]['translation_text']
|
| 52 |
-
return sentence # Return original if no model found
|
| 53 |
-
|
| 54 |
-
# Language options
|
| 55 |
-
language_options = {
|
| 56 |
-
"English": "en", "French": "fr", "Spanish": "es", "Mandarin": "zh", "Vietnamese": "vi", "Tagalog": "tl"
|
| 57 |
-
}
|
| 58 |
-
|
| 59 |
-
# Main function for Gradio
|
| 60 |
-
def process_image(image, l1, l2):
|
| 61 |
-
if not image:
|
| 62 |
-
return "Please upload an image.", None, None
|
| 63 |
-
|
| 64 |
-
caption_en = generate_caption(image)
|
| 65 |
-
caption_l2 = translate_text(caption_en, l2) if l2 != "English" else caption_en
|
| 66 |
-
|
| 67 |
-
# Generate TTS
|
| 68 |
-
tts = gTTS(text=caption_l2, lang=language_options[l2])
|
| 69 |
-
audio_path = "caption_audio.mp3"
|
| 70 |
-
tts.save(audio_path)
|
| 71 |
-
|
| 72 |
-
return caption_l2, audio_path, caption_en
|
| 73 |
-
|
| 74 |
-
# Gradio Interface
|
| 75 |
-
interface = gr.Interface(
|
| 76 |
-
fn=process_image,
|
| 77 |
-
inputs=[
|
| 78 |
-
gr.Image(type="pil", sources=["upload"]), # Only allows upload and drag-and-drop
|
| 79 |
-
gr.Dropdown(list(language_options.keys()), label="Your Native Language (L1)"),
|
| 80 |
-
gr.Dropdown(list(language_options.keys()), label="Target Language (L2)")
|
| 81 |
-
],
|
| 82 |
-
outputs=[
|
| 83 |
-
gr.Textbox(label="Generated Caption (L2)"),
|
| 84 |
-
gr.Audio(label="Listen to the Caption"),
|
| 85 |
-
gr.Textbox(label="Generated Caption (L1)")
|
| 86 |
-
],
|
| 87 |
-
title="Image Caption Generator for Language Learning",
|
| 88 |
-
description="Upload an image, select your native language (L1) and target language (L2), and get a generated caption with text-to-speech audio."
|
| 89 |
-
)
|
| 90 |
-
|
| 91 |
-
# Launch Gradio
|
| 92 |
-
interface.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|