Update app.py
Browse files
app.py
CHANGED
|
@@ -9,11 +9,12 @@ from gtts import gTTS
|
|
| 9 |
# ===============================
|
| 10 |
processor = None
|
| 11 |
caption_model = None
|
|
|
|
| 12 |
|
| 13 |
device = "cpu"
|
| 14 |
|
| 15 |
# ===============================
|
| 16 |
-
# LOAD
|
| 17 |
# ===============================
|
| 18 |
def load_blip():
|
| 19 |
global processor, caption_model
|
|
@@ -29,7 +30,16 @@ def load_blip():
|
|
| 29 |
).to(device)
|
| 30 |
|
| 31 |
# ===============================
|
| 32 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
# ===============================
|
| 34 |
def generate_caption(image):
|
| 35 |
load_blip()
|
|
@@ -41,7 +51,25 @@ def generate_caption(image):
|
|
| 41 |
return caption
|
| 42 |
|
| 43 |
# ===============================
|
| 44 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
# ===============================
|
| 46 |
def translate_and_speak(text, selected_languages):
|
| 47 |
|
|
@@ -65,7 +93,6 @@ def translate_and_speak(text, selected_languages):
|
|
| 65 |
if not code:
|
| 66 |
continue
|
| 67 |
|
| 68 |
-
# π€ Translation safe
|
| 69 |
try:
|
| 70 |
translated = GoogleTranslator(source='auto', target=code).translate(text)
|
| 71 |
except:
|
|
@@ -76,7 +103,6 @@ def translate_and_speak(text, selected_languages):
|
|
| 76 |
|
| 77 |
translations_text += f"{lang}: {translated}\n\n"
|
| 78 |
|
| 79 |
-
# π Text-to-Speech safe
|
| 80 |
try:
|
| 81 |
file_path = f"{lang}.mp3"
|
| 82 |
tts = gTTS(translated, lang=code)
|
|
@@ -93,40 +119,46 @@ def translate_and_speak(text, selected_languages):
|
|
| 93 |
def process_image(image, selected_languages):
|
| 94 |
|
| 95 |
if image is None:
|
| 96 |
-
return "β οΈ
|
| 97 |
|
| 98 |
caption = generate_caption(image)
|
|
|
|
| 99 |
|
| 100 |
translations, audios = translate_and_speak(caption, selected_languages)
|
| 101 |
|
| 102 |
-
return caption, translations, audios if audios else None
|
| 103 |
|
| 104 |
# ===============================
|
| 105 |
-
# UI
|
| 106 |
# ===============================
|
| 107 |
-
with gr.Blocks()
|
|
|
|
|
|
|
|
|
|
| 108 |
|
| 109 |
-
gr.Markdown("
|
| 110 |
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
choices=["Telugu", "Hindi", "Tamil", "Kannada", "Malayalam"],
|
| 115 |
-
label="Select Languages"
|
| 116 |
-
)
|
| 117 |
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
| 119 |
|
| 120 |
-
|
| 121 |
-
translations_out = gr.Textbox(label="Translations")
|
| 122 |
|
| 123 |
-
|
| 124 |
-
|
|
|
|
|
|
|
|
|
|
| 125 |
|
| 126 |
btn.click(
|
| 127 |
process_image,
|
| 128 |
inputs=[input_image, language_select],
|
| 129 |
-
outputs=[caption_out, translations_out, audio_out]
|
| 130 |
)
|
| 131 |
|
| 132 |
# ===============================
|
|
|
|
| 9 |
# ===============================
|
| 10 |
processor = None
|
| 11 |
caption_model = None
|
| 12 |
+
yolo_model = None
|
| 13 |
|
| 14 |
device = "cpu"
|
| 15 |
|
| 16 |
# ===============================
|
| 17 |
+
# LOAD BLIP
|
| 18 |
# ===============================
|
| 19 |
def load_blip():
|
| 20 |
global processor, caption_model
|
|
|
|
| 30 |
).to(device)
|
| 31 |
|
| 32 |
# ===============================
|
| 33 |
+
# LOAD YOLO
|
| 34 |
+
# ===============================
|
| 35 |
+
def load_yolo():
|
| 36 |
+
global yolo_model
|
| 37 |
+
if yolo_model is None:
|
| 38 |
+
from ultralytics import YOLO
|
| 39 |
+
yolo_model = YOLO("yolov8n.pt") # lightweight model
|
| 40 |
+
|
| 41 |
+
# ===============================
|
| 42 |
+
# CAPTION
|
| 43 |
# ===============================
|
| 44 |
def generate_caption(image):
|
| 45 |
load_blip()
|
|
|
|
| 51 |
return caption
|
| 52 |
|
| 53 |
# ===============================
|
| 54 |
+
# OBJECT DETECTION
|
| 55 |
+
# ===============================
|
| 56 |
+
def detect_objects(image):
|
| 57 |
+
load_yolo()
|
| 58 |
+
|
| 59 |
+
results = yolo_model(image)
|
| 60 |
+
|
| 61 |
+
labels = []
|
| 62 |
+
for r in results:
|
| 63 |
+
for box in r.boxes:
|
| 64 |
+
cls = int(box.cls[0])
|
| 65 |
+
name = yolo_model.names[cls]
|
| 66 |
+
conf = float(box.conf[0])
|
| 67 |
+
labels.append(f"{name} ({conf:.2f})")
|
| 68 |
+
|
| 69 |
+
return ", ".join(labels) if labels else "No objects detected"
|
| 70 |
+
|
| 71 |
+
# ===============================
|
| 72 |
+
# TRANSLATE + AUDIO
|
| 73 |
# ===============================
|
| 74 |
def translate_and_speak(text, selected_languages):
|
| 75 |
|
|
|
|
| 93 |
if not code:
|
| 94 |
continue
|
| 95 |
|
|
|
|
| 96 |
try:
|
| 97 |
translated = GoogleTranslator(source='auto', target=code).translate(text)
|
| 98 |
except:
|
|
|
|
| 103 |
|
| 104 |
translations_text += f"{lang}: {translated}\n\n"
|
| 105 |
|
|
|
|
| 106 |
try:
|
| 107 |
file_path = f"{lang}.mp3"
|
| 108 |
tts = gTTS(translated, lang=code)
|
|
|
|
| 119 |
def process_image(image, selected_languages):
|
| 120 |
|
| 121 |
if image is None:
|
| 122 |
+
return "β οΈ Upload image", "", "", None
|
| 123 |
|
| 124 |
caption = generate_caption(image)
|
| 125 |
+
detections = detect_objects(image)
|
| 126 |
|
| 127 |
translations, audios = translate_and_speak(caption, selected_languages)
|
| 128 |
|
| 129 |
+
return caption, detections, translations, audios if audios else None
|
| 130 |
|
| 131 |
# ===============================
|
| 132 |
+
# UI (PREMIUM)
|
| 133 |
# ===============================
|
| 134 |
+
with gr.Blocks(theme=gr.themes.Soft(), css="""
|
| 135 |
+
#title {text-align: center; font-size: 30px; font-weight: bold;}
|
| 136 |
+
.card {padding: 15px; border-radius: 15px; box-shadow: 0px 2px 10px rgba(0,0,0,0.1);}
|
| 137 |
+
""") as demo:
|
| 138 |
|
| 139 |
+
gr.Markdown("<div id='title'>π AI Vision Assistant</div>")
|
| 140 |
|
| 141 |
+
with gr.Row():
|
| 142 |
+
with gr.Column(scale=1):
|
| 143 |
+
input_image = gr.Image(type="pil", label="πΈ Upload Image")
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
+
language_select = gr.CheckboxGroup(
|
| 146 |
+
["Telugu", "Hindi", "Tamil", "Kannada", "Malayalam"],
|
| 147 |
+
label="π Select Languages"
|
| 148 |
+
)
|
| 149 |
|
| 150 |
+
btn = gr.Button("π Analyze", variant="primary")
|
|
|
|
| 151 |
|
| 152 |
+
with gr.Column(scale=1):
|
| 153 |
+
caption_out = gr.Textbox(label="π§ Caption", elem_classes="card")
|
| 154 |
+
detection_out = gr.Textbox(label="π― Detected Objects", elem_classes="card")
|
| 155 |
+
translations_out = gr.Textbox(label="π Translations", elem_classes="card")
|
| 156 |
+
audio_out = gr.File(label="π Audio Files")
|
| 157 |
|
| 158 |
btn.click(
|
| 159 |
process_image,
|
| 160 |
inputs=[input_image, language_select],
|
| 161 |
+
outputs=[caption_out, detection_out, translations_out, audio_out]
|
| 162 |
)
|
| 163 |
|
| 164 |
# ===============================
|