""" app.py ────────────────────────────────────────────────────────────────── Abnormal-Behaviour Detector (VideoMAE) Tabs ──── 1. **Video File** – upload / record → annotated video, static plot, optional beep. 2. **Live Stream** – real-time webcam with red-border overlay, rolling bar-plot, numeric prob. **Fix:** ensure live always uses the full 16-frame clips the model was trained on. """ import os import cv2 import tempfile import numpy as np import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from collections import deque import torch import gradio as gr import soundfile as sf from transformers import VideoMAEForVideoClassification, AutoImageProcessor # ── Model Initialization ───────────────────────────────────────────────── MODEL_DIR = "./model" DEVICE = "cuda" if torch.cuda.is_available() else "cpu" model = VideoMAEForVideoClassification.from_pretrained(MODEL_DIR).to(DEVICE) processor = AutoImageProcessor.from_pretrained(MODEL_DIR, use_fast=False) model.eval() # ── Constants ───────────────────────────────────────────────────────────── CLIP = 16 # frames per clip used in both upload and live STRIDE = 1 # process every new frame in live mode STREAM_FPS = 15 # polling rate for live (~15 updates/sec) HIST_LEN = 90 # rolling history length for bar plot # ── Upload / Video-file pipeline ────────────────────────────────────────── def sliding_window(path, clip_len=CLIP, stride=1): cap, frames = cv2.VideoCapture(path), [] fps = cap.get(cv2.CAP_PROP_FPS) or 25 while True: ok, f = cap.read() if not ok: break frames.append(cv2.cvtColor(f, cv2.COLOR_BGR2RGB)) cap.release() probs = [] for s in range(0, len(frames) - clip_len + 1, stride): clip = [cv2.resize(frames[i], (224,224)) for i in range(s, s + clip_len)] inp = processor(clip, return_tensors="pt").to(DEVICE) with torch.no_grad(): p = torch.softmax(model(**inp).logits, dim=1)[0,1].item() probs.append(p) return frames, probs, fps def make_outputs(frames, probs, fps, sensitivity): h, w = frames[0].shape[:2] for idx, p in enumerate(probs): if p > sensitivity: for f in range(idx, min(idx + CLIP, len(frames))): cv2.rectangle(frames[f], (0,0), (w-1,h-1), (255,0,0), 6) out_vid = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False).name vw = cv2.VideoWriter(out_vid, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w,h)) for f in frames: vw.write(cv2.cvtColor(f, cv2.COLOR_RGB2BGR)) vw.release() t = np.arange(len(probs)) / fps fig, ax = plt.subplots(figsize=(5,2)) colors = ['red' if p > sensitivity else 'blue' for p in probs] ax.bar(t, probs, color=colors, width=1/fps) ax.set_ylim(0,1); ax.set_xlabel('sec'); ax.set_ylabel('fight prob') fig.tight_layout() out_plot = tempfile.NamedTemporaryFile(suffix='.png', delete=False).name fig.savefig(out_plot, dpi=120); plt.close(fig) out_beep = None if any(p > sensitivity for p in probs): sr = 16000 tone = 0.2 * np.sin(2 * np.pi * 440 * np.linspace(0,0.5,int(sr*0.5))) out_beep = tempfile.NamedTemporaryFile(suffix='.wav', delete=False).name sf.write(out_beep, tone, sr) return out_vid, out_plot, out_beep def run_pipeline(video_path, sensitivity): if not video_path or not os.path.exists(video_path): return None, None, None frames, probs, fps = sliding_window(video_path) return make_outputs(frames, probs, fps, sensitivity) # ── Live-stream pipeline ───────────────────────────────────────────────── def plot_history(history, sensitivity): fig, ax = plt.subplots(figsize=(5,2)) colors = ['red' if p > sensitivity else 'blue' for p in history] ax.bar(np.arange(len(history)), history, color=colors) ax.set_ylim(0,1) ax.set_xlabel('recent clips'); ax.set_ylabel('fight prob') fig.tight_layout() return fig def live_handler(frame, state, history, sensitivity): state.append(frame) if len(state) < CLIP: return state, frame, history, plot_history(history, sensitivity), 0.0 clip = [cv2.resize(f, (224,224)) for f in list(state)] inp = processor(clip, return_tensors='pt').to(DEVICE) with torch.no_grad(): p = torch.softmax(model(**inp).logits, dim=1)[0,1].item() history.append(p) if len(history) > HIST_LEN: history.pop(0) disp = frame.copy() h, w = disp.shape[:2] if p > sensitivity: cv2.rectangle(disp, (0,0), (w-1,h-1), (255,0,0), 6) cv2.putText(disp, f'fight prob: {p:.2f}', (10, h-15), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255), 2, cv2.LINE_AA) return state, disp, history, plot_history(history, sensitivity), p # ── Gradio UI ─────────────────────────────────────────────────────────── with gr.Blocks(title="Abnormal Behaviour Detection") as demo: sensitivity = gr.Slider( label="Sensitivity threshold (lower → more sensitive)", minimum=0.0, maximum=1.0, step=0.01, value=0.0 ) with gr.Tab("Video File"): gr.Markdown("Upload or record a video, set sensitivity, then Analyze.") file_in = gr.Video(format='mp4', webcam_options=gr.WebcamOptions()) vid_out = gr.Video(label='Annotated Video') plot_out = gr.Image(label='Fight Probability Curve') beep_out = gr.Audio(label='Alarm Sound') gr.Button("Analyze").click( fn=run_pipeline, inputs=[file_in, sensitivity], outputs=[vid_out, plot_out, beep_out] ) with gr.Tab("Live Stream"): gr.Markdown( "Live webcam. Border & plot turn red when `prob > sensitivity`." ) cam_in = gr.Image(sources='webcam', streaming=True, type='numpy', label='Webcam') cam_out = gr.Image(streaming=True, label='Annotated Feed') plot_live = gr.Plot(label='Rolling Fight Probability') num_prob = gr.Number(label='Current Probability', precision=2) frame_state = gr.State(value=deque(maxlen=CLIP)) prob_state = gr.State(value=[]) cam_in.stream( fn=live_handler, inputs=[cam_in, frame_state, prob_state, sensitivity], outputs=[frame_state, cam_out, prob_state, plot_live, num_prob], stream_every=1/STREAM_FPS ) demo.queue().launch()