Robin7339's picture
Update app.py
aa8516d verified
Raw History Blame Contribute Delete
33.1 kB
"""
IRIS β€” Intelligent Real-time Inference System for Visual Assistance
Next-Gen Cyber Assistive HUD & Real-time Spatial Navigation
Gradio Application for Hugging Face Spaces (Compatible with ZeroGPU & CPU)
"""
import os
import base64
import tempfile
from pathlib import Path
# Redirect YOLO config/cache to writable directory
os.environ["YOLO_CONFIG_DIR"] = "/tmp/Ultralytics"
# Hugging Face ZeroGPU compatibility
try:
import spaces
except ImportError:
class spaces:
@staticmethod
def GPU(func=None, duration=60):
if func is None:
def decorator(f):
return f
return decorator
return func
import cv2
import numpy as np
from PIL import Image
from ultralytics import YOLO
from gtts import gTTS
import gradio as gr
from priority_engine import PriorityEngine
# ── Config ───────────────────────────────────────────────────────────────────
CONF_THRESHOLD = 0.35
MODEL_NAME = "yolo11n.pt"
# Initialize model & priority engine
model = YOLO(MODEL_NAME)
priority = PriorityEngine()
# Colors for bounding boxes (RGB)
POSITION_COLORS = {
"left": (255, 140, 0), # Orange
"center": (16, 185, 129), # Emerald Green (Directly Ahead)
"right": (59, 130, 246), # Electric Blue
}
def get_position(center_x, frame_width):
if center_x < frame_width / 3:
return "left"
elif center_x < 2 * frame_width / 3:
return "center"
return "right"
def generate_tts_audio(text: str):
"""Generate an MP3 audio file using gTTS."""
if not text or len(text.strip()) == 0:
return None
try:
clean_text = (
text.replace("πŸ”Š", "")
.replace("⚠️", "")
.replace("β€”", ", ")
.replace("–", ", ")
.strip()
)
tts = gTTS(text=clean_text, lang="en", slow=False)
tmp = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
tts.save(tmp.name)
return tmp.name
except Exception as e:
print(f"[TTS Error] {e}")
return None
def render_spatial_hud(detections):
"""Render a 3-zone spatial radar HUD and obstacle badges."""
left_items = [d for d in detections if d["position"] == "left"]
center_items = [d for d in detections if d["position"] == "center"]
right_items = [d for d in detections if d["position"] == "right"]
def make_zone(items, zone_name, color):
if not items:
return f"""
<div class="hud-zone" style="border-top: 3px solid {color}33;">
<div class="hud-zone-header">
<span class="hud-zone-dot" style="background:{color};"></span>
<span class="hud-zone-name">{zone_name}</span>
</div>
<div class="hud-zone-val val-clear">CLEAR</div>
</div>
"""
top_obj = items[0]["object"].capitalize()
conf = int(items[0]["confidence"] * 100)
extra = f" +{len(items)-1}" if len(items) > 1 else ""
is_warn = zone_name == "CENTER PATH"
val_cls = "val-alert" if is_warn else "val-warn"
return f"""
<div class="hud-zone active" style="border-top: 3px solid {color};">
<div class="hud-zone-header">
<span class="hud-zone-dot pulse" style="background:{color};"></span>
<span class="hud-zone-name">{zone_name}</span>
</div>
<div class="hud-zone-val {val_cls}">{top_obj} ({conf}%){extra}</div>
</div>
"""
zones_html = f"""
<div class="hud-zones-bar">
{make_zone(left_items, "LEFT ZONE", "#f97316")}
{make_zone(center_items, "CENTER PATH", "#10b981")}
{make_zone(right_items, "RIGHT ZONE", "#3b82f6")}
</div>
"""
if detections:
badges = []
for d in detections:
c = POSITION_COLORS.get(d["position"], (200, 200, 200))
rgb_str = f"rgb({c[0]},{c[1]},{c[2]})"
badges.append(f"""
<div class="det-chip">
<span class="det-indicator" style="background:{rgb_str}; box-shadow: 0 0 8px {rgb_str}aa;"></span>
<span class="det-title">{d['object'].capitalize()}</span>
<span class="det-dir" style="color:{rgb_str};">{d['position'].upper()}</span>
<span class="det-score">{int(d['confidence']*100)}%</span>
</div>
""")
chips_html = f"""
<div class="hud-breakdown">
<div class="hud-subhead">RADAR DETECTIONS ({len(detections)})</div>
<div class="chips-flex">
{"".join(badges)}
</div>
</div>
"""
else:
chips_html = """
<div class="hud-breakdown">
<div class="hud-subhead">RADAR DETECTIONS</div>
<div class="all-clear-banner">
<span class="check-icon">βœ“</span> Navigation corridor is clear of detected obstacles.
</div>
</div>
"""
return zones_html + chips_html
@spaces.GPU
def analyze_frame(image):
"""
Run YOLO inference and Priority Engine on an input image.
Returns:
annotated_image, instruction_text, audio_filepath, detections_hud_html
"""
if image is None:
placeholder = """
<div class="hud-zones-bar">
<div class="hud-zone"><div class="hud-zone-val val-clear">STANDBY</div></div>
</div>
"""
return None, "STANDBY β€” Waiting for visual input.", None, placeholder
h, w, _ = image.shape
results = model(image, conf=CONF_THRESHOLD, verbose=False)
detections = []
annotated = image.copy()
for result in results:
for box in result.boxes:
conf = float(box.conf[0])
if conf < CONF_THRESHOLD:
continue
cls_id = int(box.cls[0])
label = model.names[cls_id]
x1, y1, x2, y2 = box.xyxy[0].tolist()
center_x = (x1 + x2) / 2
pos = get_position(center_x, w)
detections.append({
"object": label,
"confidence": round(conf, 2),
"position": pos,
"bbox": [round(x1, 1), round(y1, 1), round(x2, 1), round(y2, 1)],
})
# Draw bounding box and label in RGB
color = POSITION_COLORS.get(pos, (255, 255, 255))
cv2.rectangle(annotated, (int(x1), int(y1)), (int(x2), int(y2)), color, 2)
tag = f"{label.upper()} {int(conf * 100)}%"
(tw, th), _ = cv2.getTextSize(tag, cv2.FONT_HERSHEY_SIMPLEX, 0.55, 2)
ty = max(int(y1) - 8, th + 6)
cv2.rectangle(annotated, (int(x1), ty - th - 4), (int(x1) + tw + 6, ty + 2), color, -1)
cv2.putText(
annotated, tag, (int(x1) + 3, ty - 2),
cv2.FONT_HERSHEY_SIMPLEX, 0.55, (10, 15, 25), 2
)
detections.sort(key=lambda d: d["confidence"], reverse=True)
instruction = priority.pick(detections)
audio_path = generate_tts_audio(instruction)
hud_html = render_spatial_hud(detections)
return annotated, f"πŸ”Š {instruction}", audio_path, hud_html
@spaces.GPU
def analyze_webcam_input(b64_data, fallback_image):
"""Decodes live frame from browser canvas base64 or fallback Gradio snapshot."""
image = None
if b64_data and len(b64_data) > 100:
try:
raw = base64.b64decode(b64_data.split(",")[-1])
arr = np.frombuffer(raw, np.uint8)
bgr = cv2.imdecode(arr, cv2.IMREAD_COLOR)
if bgr is not None:
image = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
except Exception as e:
print(f"[Live frame decode error] {e}")
if image is None and fallback_image is not None:
image = fallback_image
if image is None:
placeholder = """
<div class="all-clear-banner" style="color: #f59e0b; border-color: rgba(245,158,11,0.3);">
⚠️ Camera stream inactive. Click "Start Camera" above.
</div>
"""
return None, "STANDBY β€” Please start the camera stream.", None, placeholder
return analyze_frame(image)
@spaces.GPU(duration=120)
def analyze_video(video_path):
"""Sample key frames from an uploaded video and generate navigation guidance."""
if not video_path:
return None, "No video provided", None
cap = cv2.VideoCapture(video_path)
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = cap.get(cv2.CAP_PROP_FPS) or 30
sample_step = max(1, total // 6)
sampled_frames = []
frame_idx = 0
all_instructions = []
while True:
ret, frame = cap.read()
if not ret:
break
if frame_idx % sample_step == 0:
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
ann, inst, _, _ = analyze_frame(frame_rgb)
sampled_frames.append(ann)
sec = round(frame_idx / fps, 1)
all_instructions.append(f"⏱️ **{sec}s**: {inst.replace('πŸ”Š ', '')}")
frame_idx += 1
cap.release()
timeline = "\n\n".join(all_instructions)
first_instruction = all_instructions[0].split(": ")[-1] if all_instructions else "Video analysis complete."
overall_tts = generate_tts_audio(first_instruction)
return sampled_frames, timeline, overall_tts
# ── Sequential Live Navigation Controller (Speaks 1st, then scans next) ─────
live_nav_js = """
() => {
if (typeof window.isLiveNavActive === 'undefined') {
window.isLiveNavActive = false;
window.liveNavTimer = null;
}
window.isLiveNavActive = !window.isLiveNavActive;
const btn = document.querySelector("#live_nav_toggle_btn button, #live_nav_toggle_btn");
const status = document.getElementById("live_status_indicator");
function triggerScan() {
if (!window.isLiveNavActive) return;
const scanBtn = document.querySelector("#scan_frame_btn button, #scan_frame_btn");
if (scanBtn && !scanBtn.disabled) {
scanBtn.click();
}
}
// Called as soon as YOLO finishes processing a frame
window.onFrameOutputReceived = function(instructionText) {
if (!instructionText) {
if (window.isLiveNavActive) {
window.liveNavTimer = setTimeout(triggerScan, 1000);
}
return;
}
const clean = instructionText.replace("πŸ”Š", "").replace("⚠️", "").replace("STANDBY", "").trim();
if (!clean || clean.includes("Please start") || clean.includes("Waiting for") || clean.includes("Camera stream")) {
if (window.isLiveNavActive) {
window.liveNavTimer = setTimeout(triggerScan, 1200);
}
return;
}
if (!('speechSynthesis' in window)) {
// Fallback if browser doesn't support Web Speech
if (window.isLiveNavActive) {
window.liveNavTimer = setTimeout(triggerScan, 2500);
}
return;
}
// Cancel previous utterance
window.speechSynthesis.cancel();
const u = new SpeechSynthesisUtterance(clean);
u.rate = 0.95;
u.pitch = 1.0;
let hasFinished = false;
function onSpeechComplete() {
if (hasFinished) return;
hasFinished = true;
// Frame output has finished speaking!
// Wait 500ms pause, then trigger the next frame scan:
if (window.isLiveNavActive) {
window.liveNavTimer = setTimeout(triggerScan, 500);
}
}
u.onend = onSpeechComplete;
u.onerror = onSpeechComplete;
// Safety timeout in case browser onend event is dropped
const safeTimeout = Math.max(3500, clean.length * 120);
setTimeout(onSpeechComplete, safeTimeout);
window.speechSynthesis.speak(u);
};
if (window.isLiveNavActive) {
if (btn) {
btn.innerText = "⏹️ Stop Real-Time Navigation";
btn.style.setProperty("background", "linear-gradient(135deg, #ef4444, #dc2626)", "important");
btn.classList.add("active");
}
if (status) {
status.innerHTML = "<span style='color:#10b981; font-weight:700;'>● ACTIVE</span> β€” Sequential speech: speaks full alert before scanning next frame.";
}
// Trigger the initial frame scan
triggerScan();
} else {
if (btn) {
btn.innerText = "▢️ Start Real-Time Navigation (Sequential Voice)";
btn.style.setProperty("background", "linear-gradient(135deg, #10b981, #059669)", "important");
btn.classList.remove("active");
}
if (status) {
status.innerHTML = "<span style='color:#64748b;'>● STANDBY</span> β€” Click above to begin continuous scanning.";
}
if (window.liveNavTimer) {
clearTimeout(window.liveNavTimer);
window.liveNavTimer = null;
}
if ('speechSynthesis' in window) {
window.speechSynthesis.cancel();
}
}
}
"""
# Extract current live frame from HTML5 video element with no shutter lag
js_extract_video = """
(b64, fallback_img) => {
const video = document.querySelector("#webcam_viewport video");
if (video && video.videoWidth > 0) {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const ctx = canvas.getContext("2d");
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
return [canvas.toDataURL("image/jpeg", 0.85), null];
}
return [b64, fallback_img];
}
"""
# ── Ultra-Professional Dark Theme CSS ────────────────────────────────────────
custom_css = """
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Plus+Jakarta+Sans:wght@400;500;600;700;800&family=JetBrains+Mono:wght@500;700&display=swap" rel="stylesheet">
<style>
:root {
--iris-bg: #07090e;
--iris-card: rgba(15, 18, 28, 0.75);
--iris-border: rgba(255, 255, 255, 0.08);
--iris-cyan: #06b6d4;
--iris-indigo: #6366f1;
--iris-emerald: #10b981;
--iris-amber: #f59e0b;
--iris-rose: #f43f5e;
}
body, .gradio-container {
background-color: var(--iris-bg) !important;
font-family: 'Plus Jakarta Sans', -apple-system, BlinkMacSystemFont, sans-serif !important;
color: #f1f5f9 !important;
max-width: 1200px !important;
margin: 0 auto !important;
}
/* Header & HUD Navbar */
.iris-brand-hero {
display: flex;
align-items: center;
justify-content: space-between;
padding: 18px 24px;
margin-bottom: 20px;
background: linear-gradient(135deg, rgba(20, 25, 40, 0.85), rgba(10, 13, 22, 0.95));
border: 1px solid var(--iris-border);
border-radius: 20px;
backdrop-filter: blur(20px);
box-shadow: 0 10px 30px -10px rgba(0, 0, 0, 0.5), inset 0 1px 0 rgba(255, 255, 255, 0.1);
}
.brand-left {
display: flex;
align-items: center;
gap: 16px;
}
.brand-logo-hex {
width: 46px;
height: 46px;
border-radius: 14px;
background: linear-gradient(135deg, #6366f1, #06b6d4);
display: flex;
align-items: center;
justify-content: center;
box-shadow: 0 0 20px rgba(99, 102, 241, 0.5);
}
.brand-text h1 {
font-size: 1.5rem;
font-weight: 800;
letter-spacing: -0.02em;
background: linear-gradient(135deg, #ffffff 40%, #a5b4fc 100%);
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
margin: 0;
line-height: 1.2;
}
.brand-text p {
font-size: 0.8rem;
color: #94a3b8;
margin: 0;
font-weight: 500;
letter-spacing: 0.05em;
text-transform: uppercase;
}
.brand-badges {
display: flex;
gap: 8px;
}
.tech-badge {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 6px 12px;
border-radius: 9999px;
font-size: 0.75rem;
font-weight: 600;
font-family: 'JetBrains Mono', monospace;
background: rgba(255, 255, 255, 0.05);
border: 1px solid var(--iris-border);
color: #cbd5e1;
}
.badge-pulse {
width: 8px;
height: 8px;
border-radius: 50%;
background: var(--iris-emerald);
box-shadow: 0 0 10px var(--iris-emerald);
animation: pulseGlow 1.8s infinite;
}
@keyframes pulseGlow {
0%, 100% { transform: scale(1); opacity: 1; }
50% { transform: scale(1.3); opacity: 0.7; }
}
/* Tabs */
.tabs {
border: none !important;
background: transparent !important;
}
.tab-nav {
border: none !important;
background: rgba(15, 20, 32, 0.6) !important;
padding: 6px !important;
border-radius: 14px !important;
margin-bottom: 20px !important;
border: 1px solid var(--iris-border) !important;
display: flex !important;
gap: 6px !important;
}
.tab-nav button {
border-radius: 10px !important;
font-weight: 600 !important;
font-size: 0.9rem !important;
padding: 10px 18px !important;
color: #94a3b8 !important;
transition: all 0.2s ease !important;
border: none !important;
background: transparent !important;
}
.tab-nav button.selected {
background: linear-gradient(135deg, rgba(99, 102, 241, 0.25), rgba(6, 182, 212, 0.2)) !important;
color: #ffffff !important;
border: 1px solid rgba(99, 102, 241, 0.4) !important;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3) !important;
}
/* Glassmorphic Panels */
.block {
background: var(--iris-card) !important;
border: 1px solid var(--iris-border) !important;
border-radius: 18px !important;
backdrop-filter: blur(12px) !important;
}
/* Urgent Instruction Banner */
.instruction-box textarea {
font-size: 1.35rem !important;
font-weight: 800 !important;
line-height: 1.4 !important;
color: #38bdf8 !important;
background: linear-gradient(135deg, #0c1527, #131c31) !important;
border: 2px solid rgba(56, 189, 248, 0.4) !important;
border-radius: 14px !important;
text-align: center !important;
box-shadow: 0 0 25px rgba(56, 189, 248, 0.15) !important;
padding: 16px !important;
}
/* Action Buttons */
.btn-primary-action {
background: linear-gradient(135deg, #6366f1, #4f46e5) !important;
color: white !important;
font-weight: 700 !important;
font-size: 1rem !important;
border: 1px solid rgba(255, 255, 255, 0.15) !important;
border-radius: 12px !important;
padding: 12px 20px !important;
box-shadow: 0 4px 16px rgba(99, 102, 241, 0.35) !important;
transition: all 0.2s ease !important;
}
.btn-primary-action:hover {
transform: translateY(-1px) !important;
box-shadow: 0 6px 20px rgba(99, 102, 241, 0.5) !important;
}
/* Live Navigation Toggle Button */
#live_nav_toggle_btn, .live-stream-btn {
width: 100% !important;
padding: 14px 20px !important;
border: 1px solid rgba(255, 255, 255, 0.15) !important;
border-radius: 12px !important;
background: linear-gradient(135deg, #10b981, #059669) !important;
color: white !important;
font-weight: 700 !important;
font-size: 1.05rem !important;
cursor: pointer !important;
box-shadow: 0 4px 16px rgba(16, 185, 129, 0.35) !important;
transition: all 0.2s ease !important;
}
#live_nav_toggle_btn:hover {
filter: brightness(1.1) !important;
transform: translateY(-1px) !important;
}
#live_nav_toggle_btn.active {
background: linear-gradient(135deg, #ef4444, #dc2626) !important;
box-shadow: 0 4px 20px rgba(239, 68, 68, 0.5) !important;
animation: livePulse 2s infinite;
}
@keyframes livePulse {
0%, 100% { box-shadow: 0 0 15px rgba(239, 68, 68, 0.5); }
50% { box-shadow: 0 0 25px rgba(239, 68, 68, 0.8); }
}
/* Spatial Radar HUD */
.hud-zones-bar {
display: grid;
grid-template-columns: repeat(3, 1fr);
gap: 10px;
margin-bottom: 14px;
}
.hud-zone {
background: rgba(10, 15, 26, 0.7);
border: 1px solid var(--iris-border);
border-radius: 12px;
padding: 10px 12px;
transition: all 0.2s ease;
}
.hud-zone-header {
display: flex;
align-items: center;
gap: 6px;
margin-bottom: 4px;
}
.hud-zone-dot {
width: 7px;
height: 7px;
border-radius: 50%;
}
.hud-zone-name {
font-size: 0.7rem;
font-weight: 700;
letter-spacing: 0.05em;
color: #94a3b8;
}
.hud-zone-val {
font-family: 'JetBrains Mono', monospace;
font-size: 0.88rem;
font-weight: 700;
}
.val-clear { color: #10b981; }
.val-warn { color: #f59e0b; }
.val-alert { color: #ef4444; }
/* Detection Chips */
.hud-breakdown {
background: rgba(10, 15, 26, 0.5);
border: 1px solid var(--iris-border);
border-radius: 14px;
padding: 12px 14px;
}
.hud-subhead {
font-size: 0.7rem;
font-weight: 700;
letter-spacing: 0.08em;
color: #64748b;
margin-bottom: 8px;
}
.chips-flex {
display: flex;
flex-wrap: wrap;
gap: 8px;
}
.det-chip {
display: inline-flex;
align-items: center;
gap: 8px;
background: rgba(255, 255, 255, 0.04);
border: 1px solid var(--iris-border);
border-radius: 8px;
padding: 6px 10px;
font-size: 0.8rem;
}
.det-indicator {
width: 6px;
height: 6px;
border-radius: 50%;
}
.det-title {
font-weight: 600;
color: #f1f5f9;
}
.det-dir {
font-family: 'JetBrains Mono', monospace;
font-size: 0.7rem;
font-weight: 700;
}
.det-score {
font-family: 'JetBrains Mono', monospace;
font-size: 0.72rem;
color: #94a3b8;
}
.all-clear-banner {
display: flex;
align-items: center;
gap: 8px;
color: #10b981;
font-size: 0.85rem;
font-weight: 600;
padding: 6px 0;
}
.check-icon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 18px;
height: 18px;
border-radius: 50%;
background: rgba(16, 185, 129, 0.2);
font-size: 0.75rem;
}
</style>
"""
# ── Build Gradio Interface ───────────────────────────────────────────────────
with gr.Blocks(title="IRIS β€” Intelligent Visual Copilot") as demo:
gr.HTML(custom_css)
# Top Brand Bar
gr.HTML("""
<div class="iris-brand-hero">
<div class="brand-left">
<div class="brand-logo-hex">
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="white" stroke-width="2.5">
<circle cx="12" cy="12" r="3"></circle>
<path d="M2.05 12a10 10 0 0 1 19.9 0 10 10 0 0 1-19.9 0Z"></path>
</svg>
</div>
<div class="brand-text">
<h1>IRIS VISUAL COPILOT</h1>
<p>Real-Time Spatial Obstacle Detection & Voice Guidance</p>
</div>
</div>
<div class="brand-badges">
<div class="tech-badge">
<span class="badge-pulse"></span>
<span>SYSTEM ONLINE</span>
</div>
<div class="tech-badge">
<span>YOLO11-NANO</span>
</div>
<div class="tech-badge">
<span>3-ZONE RADAR</span>
</div>
</div>
</div>
""")
with gr.Tabs():
# ── TAB 1: Real-Time Live Vision ─────────────────────────────────────
with gr.TabItem("πŸ“· Live Visual Navigation", id="tab_live"):
with gr.Row():
with gr.Column(scale=5):
webcam_input = gr.Image(
sources=["webcam"],
type="numpy",
label="Live Optical Sensor",
elem_id="webcam_viewport",
)
webcam_b64 = gr.Textbox(visible=False, elem_id="webcam_b64_buffer")
# Native Gradio button for live navigation toggle
live_nav_btn = gr.Button(
"▢️ Start Real-Time Navigation (Sequential Voice)",
elem_id="live_nav_toggle_btn",
elem_classes=["live-stream-btn"],
variant="primary",
)
gr.HTML("""
<p id="live_status_indicator" style="text-align: center; font-size: 0.82rem; color: #94a3b8; margin: 6px 0 12px 0;">
<span style="color:#64748b;">● STANDBY</span> β€” Click above to begin continuous scanning.
</p>
""")
webcam_snap_btn = gr.Button(
"⚑ Instant Scan (Current Frame)",
elem_id="scan_frame_btn",
elem_classes=["btn-primary-action"],
variant="secondary",
)
with gr.Column(scale=6):
webcam_instruction = gr.Textbox(
label="πŸ“’ Priority Voice Navigation Alert",
interactive=False,
elem_id="webcam_instruction",
elem_classes=["instruction-box"],
value="STANDBY β€” Waiting for visual input.",
)
webcam_audio = gr.Audio(
label="πŸ”Š Audio Recording (Optional)",
autoplay=False,
)
webcam_hud = gr.HTML(
label="Spatial Radar Breakdown",
value="""
<div class="hud-zones-bar">
<div class="hud-zone"><div class="hud-zone-val val-clear">STANDBY</div></div>
</div>
""",
)
webcam_output = gr.Image(
label="Annotated Spatial View",
)
# Wire up the instant scan button (extracts live frame via JS)
# When inference completes, .then() triggers sequential voice & next scan
webcam_snap_btn.click(
fn=analyze_webcam_input,
inputs=[webcam_b64, webcam_input],
outputs=[webcam_output, webcam_instruction, webcam_audio, webcam_hud],
js=js_extract_video,
).then(
fn=None,
inputs=[webcam_instruction],
js="""(inst) => { if (window.onFrameOutputReceived) { window.onFrameOutputReceived(inst); } }""",
)
# Wire up the live navigation button to toggle the continuous loop
live_nav_btn.click(
fn=None,
js=live_nav_js,
)
# ── TAB 2: Image Diagnostic ──────────────────────────────────────────
with gr.TabItem("πŸ–ΌοΈ Photo Inspection", id="tab_photo"):
with gr.Row():
with gr.Column(scale=5):
upload_input = gr.Image(
sources=["upload"],
type="numpy",
label="Upload Scene Photograph",
)
upload_btn = gr.Button(
"πŸ” Analyze Scene Obstacles",
elem_classes=["btn-primary-action"],
variant="primary",
)
with gr.Column(scale=6):
upload_instruction = gr.Textbox(
label="πŸ“’ Spoken Navigation Instruction",
interactive=False,
elem_classes=["instruction-box"],
)
upload_audio = gr.Audio(
label="πŸ”Š Audio Instruction",
autoplay=True,
)
upload_hud = gr.HTML(
label="Spatial Radar Breakdown",
)
upload_output = gr.Image(
label="Annotated Spatial View",
)
upload_btn.click(
fn=analyze_frame,
inputs=[upload_input],
outputs=[upload_output, upload_instruction, upload_audio, upload_hud],
)
# ── TAB 3: Video Walkthrough ─────────────────────────────────────────
with gr.TabItem("πŸŽ₯ Video Walkthrough", id="tab_video"):
with gr.Row():
with gr.Column(scale=5):
video_input = gr.Video(
sources=["upload"],
label="Upload Navigation Footage (.mp4, .mov)",
)
video_btn = gr.Button(
"🎬 Analyze Navigation Corridor",
elem_classes=["btn-primary-action"],
variant="primary",
)
with gr.Column(scale=6):
video_audio = gr.Audio(
label="πŸ”Š Spoken Overview",
autoplay=True,
)
video_timeline = gr.Markdown(
label="Chronological Timeline Guidance",
)
video_gallery = gr.Gallery(
label="Corridor Keyframe Snapshots",
columns=2,
)
video_btn.click(
fn=analyze_video,
inputs=[video_input],
outputs=[video_gallery, video_timeline, video_audio],
)
# ── TAB 4: System Architecture ───────────────────────────────────────
with gr.TabItem("ℹ️ System Architecture", id="tab_about"):
gr.HTML("""
<div style="background: rgba(15, 20, 32, 0.6); border: 1px solid rgba(255, 255, 255, 0.08); border-radius: 16px; padding: 24px;">
<h2 style="color: #f1f5f9; font-size: 1.3rem; margin-bottom: 12px;">🧭 IRIS Navigation Engine Principles</h2>
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; margin-top: 16px;">
<div style="background: rgba(255, 255, 255, 0.03); border: 1px solid rgba(255, 255, 255, 0.06); border-radius: 12px; padding: 16px;">
<h3 style="color: #38bdf8; font-size: 1rem; margin-bottom: 6px;">1. 3-Zone Spatial Radar</h3>
<p style="color: #94a3b8; font-size: 0.85rem; line-height: 1.5; margin: 0;">
The sensor field is dynamically partitioned into <b>LEFT (33%)</b>, <b>CENTER PATH (33%)</b>, and <b>RIGHT (33%)</b>. Obstacles located directly in the center path receive critical priority.
</p>
</div>
<div style="background: rgba(255, 255, 255, 0.03); border: 1px solid rgba(255, 255, 255, 0.06); border-radius: 12px; padding: 16px;">
<h3 style="color: #10b981; font-size: 1rem; margin-bottom: 6px;">2. Zero-Cognitive-Overload Priority</h3>
<p style="color: #94a3b8; font-size: 0.85rem; line-height: 1.5; margin: 0;">
Instead of overwhelming a visually impaired person with a list of 10 items, IRIS selects <b>exactly ONE</b> urgent, actionable instruction (e.g. <i>"Caution! Person directly ahead"</i> or <i>"Watch out β€” chair ahead. Step around it"</i>).
</p>
</div>
<div style="background: rgba(255, 255, 255, 0.03); border: 1px solid rgba(255, 255, 255, 0.06); border-radius: 12px; padding: 16px;">
<h3 style="color: #a855f7; font-size: 1rem; margin-bottom: 6px;">3. Dual-Stream Voice Guidance</h3>
<p style="color: #94a3b8; font-size: 0.85rem; line-height: 1.5; margin: 0;">
Low-latency browser speech synthesis provides instant voice cues on device, complemented by server-side synthesized audio with automatic playback.
</p>
</div>
</div>
</div>
""")
if __name__ == "__main__":
demo.launch()