Update app.py
Browse files
app.py
CHANGED
|
@@ -1,37 +1,40 @@
|
|
| 1 |
"""
|
| 2 |
-
app.py
|
| 3 |
-
|
| 4 |
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
|
| 8 |
-
# βββββββββββββββββββββββββββββ
|
| 9 |
-
import os, cv2, tempfile, numpy as np, matplotlib
|
| 10 |
-
|
|
|
|
|
|
|
| 11 |
|
| 12 |
import torch, gradio as gr, soundfile as sf
|
| 13 |
-
from
|
| 14 |
-
|
| 15 |
-
#
|
| 16 |
-
|
| 17 |
-
from fastrtc import WebRTC
|
| 18 |
-
webrtc_ok = True
|
| 19 |
-
except ModuleNotFoundError:
|
| 20 |
-
try:
|
| 21 |
-
from gradio_webrtc import WebRTC
|
| 22 |
-
webrtc_ok = True
|
| 23 |
-
except ModuleNotFoundError:
|
| 24 |
-
webrtc_ok = False
|
| 25 |
-
|
| 26 |
-
# βββββββββββββββββββββββββββ model load ββββββββββββββββββββββββββββ
|
| 27 |
-
MODEL_DIR = "./model"
|
| 28 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 29 |
|
| 30 |
model = VideoMAEForVideoClassification.from_pretrained(MODEL_DIR).to(DEVICE)
|
| 31 |
processor = AutoImageProcessor.from_pretrained(MODEL_DIR, use_fast=False)
|
| 32 |
model.eval()
|
| 33 |
|
| 34 |
-
# βββββββββββββββββββββ
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
def sliding_window(path, clip_len=16, stride=4):
|
| 36 |
cap, frames = cv2.VideoCapture(path), []
|
| 37 |
fps = cap.get(cv2.CAP_PROP_FPS) or 25
|
|
@@ -44,48 +47,51 @@ def sliding_window(path, clip_len=16, stride=4):
|
|
| 44 |
|
| 45 |
probs, boxes = [], []
|
| 46 |
for s in range(0, len(frames) - clip_len, stride):
|
| 47 |
-
clip = [cv2.resize(frames[i], (224, 224))
|
|
|
|
| 48 |
inp = processor(clip, return_tensors="pt").to(DEVICE)
|
| 49 |
with torch.no_grad():
|
| 50 |
p = torch.softmax(model(**inp).logits, dim=1)[0, 1].item()
|
| 51 |
probs.append(p)
|
| 52 |
-
if p >
|
| 53 |
boxes.append((s / fps, (s + clip_len) / fps))
|
| 54 |
return frames, probs, boxes, fps
|
| 55 |
|
| 56 |
-
|
| 57 |
-
|
| 58 |
h, w, _ = frames[0].shape
|
| 59 |
for t1, t2 in boxes:
|
| 60 |
for i in range(int(t1 * fps), min(int(t2 * fps), len(frames))):
|
| 61 |
cv2.rectangle(frames[i], (0, 0), (w - 1, h - 1), (255, 0, 0), 6)
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
|
|
|
| 65 |
for f in frames:
|
| 66 |
vw.write(cv2.cvtColor(f, cv2.COLOR_RGB2BGR))
|
| 67 |
vw.release()
|
| 68 |
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
|
|
|
| 79 |
|
|
|
|
| 80 |
alarm = None
|
| 81 |
-
if any(p >
|
| 82 |
sr = 16_000
|
| 83 |
t_beep = np.linspace(0, 0.5, int(sr * 0.5), endpoint=False)
|
| 84 |
alarm = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
|
| 85 |
sf.write(alarm, 0.2 * np.sin(2 * np.pi * 440 * t_beep), sr)
|
| 86 |
|
| 87 |
-
return
|
| 88 |
-
|
| 89 |
|
| 90 |
def run_pipeline(file_path):
|
| 91 |
if not file_path or not os.path.exists(file_path):
|
|
@@ -93,13 +99,9 @@ def run_pipeline(file_path):
|
|
| 93 |
frames, probs, boxes, fps = sliding_window(file_path)
|
| 94 |
return make_outputs(frames, probs, boxes, fps)
|
| 95 |
|
| 96 |
-
# ββββββββββββββββββββββ
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def make_plot(probs): # helper to draw the bar plot
|
| 100 |
-
import matplotlib.pyplot as plt
|
| 101 |
-
import numpy as np
|
| 102 |
-
|
| 103 |
fig, ax = plt.subplots(figsize=(5, 2))
|
| 104 |
colors = ["red" if p > THRESH else "blue" for p in probs]
|
| 105 |
ax.bar(np.arange(len(probs)), probs, color=colors)
|
|
@@ -111,27 +113,26 @@ def make_plot(probs): # helper to draw the bar plot
|
|
| 111 |
|
| 112 |
def live_handler(frame, state, probs):
|
| 113 |
"""
|
| 114 |
-
frame
|
| 115 |
-
state
|
| 116 |
-
probs
|
| 117 |
-
returns: updated state, annotated frame, updated probs, plot-figure
|
| 118 |
"""
|
| 119 |
state.append(frame)
|
| 120 |
-
if len(state) < CLIP:
|
| 121 |
return state, frame, probs, make_plot(probs)
|
| 122 |
|
| 123 |
-
if live_handler.i % STRIDE == 0:
|
| 124 |
clip = [cv2.resize(f, (224, 224)) for f in list(state)]
|
| 125 |
inp = processor(clip, return_tensors="pt").to(DEVICE)
|
| 126 |
with torch.no_grad():
|
| 127 |
-
p = torch.softmax(model(**inp).logits,
|
| 128 |
-
|
| 129 |
-
probs.append(p)
|
| 130 |
-
if len(probs) > 60:
|
| 131 |
probs.pop(0)
|
| 132 |
live_handler.i += 1
|
| 133 |
|
| 134 |
-
#
|
| 135 |
disp = frame.copy()
|
| 136 |
h, w = disp.shape[:2]
|
| 137 |
if live_handler.p > THRESH:
|
|
@@ -144,70 +145,52 @@ def live_handler(frame, state, probs):
|
|
| 144 |
|
| 145 |
live_handler.i, live_handler.p = 0, 0.0
|
| 146 |
|
| 147 |
-
#
|
| 148 |
-
buffer_webrtc = deque(maxlen=CLIP)
|
| 149 |
-
|
| 150 |
-
def live_handler_webrtc(frame):
|
| 151 |
-
buffer_webrtc.append(frame)
|
| 152 |
-
if len(buffer_webrtc) < CLIP:
|
| 153 |
-
return frame
|
| 154 |
-
|
| 155 |
-
if live_handler_webrtc.i % STRIDE == 0:
|
| 156 |
-
clip = [cv2.resize(f, (224, 224)) for f in list(buffer_webrtc)]
|
| 157 |
-
inp = processor(clip, return_tensors="pt").to(DEVICE)
|
| 158 |
-
with torch.no_grad():
|
| 159 |
-
live_handler_webrtc.p = torch.softmax(
|
| 160 |
-
model(**inp).logits, dim=1
|
| 161 |
-
)[0, 1].item()
|
| 162 |
-
live_handler_webrtc.i += 1
|
| 163 |
-
|
| 164 |
-
disp = frame.copy()
|
| 165 |
-
h, w = disp.shape[:2]
|
| 166 |
-
if live_handler_wertc.p > THRESH:
|
| 167 |
-
cv2.rectangle(disp, (0, 0), (w - 1, h - 1), (255, 0, 0), 6)
|
| 168 |
-
cv2.putText(disp, f"fight prob: {live_handler_webrtc.p:.2f}",
|
| 169 |
-
(10, h - 15), cv2.FONT_HERSHEY_SIMPLEX, 0.7,
|
| 170 |
-
(255, 255, 255), 2, cv2.LINE_AA)
|
| 171 |
-
return disp
|
| 172 |
-
|
| 173 |
-
live_handler_webrtc.i, live_handler_webrtc.p = 0, 0.0
|
| 174 |
-
|
| 175 |
-
# βββββββββββββββββββββββββββββ UI βββββββββββββββββββββββββββββββββββ
|
| 176 |
with gr.Blocks(title="Abnormal Behaviour Detection") as demo:
|
| 177 |
|
| 178 |
-
#
|
| 179 |
with gr.Tab("Video file"):
|
| 180 |
-
gr.Markdown(
|
|
|
|
|
|
|
|
|
|
| 181 |
file_in = gr.Video(label="Upload / record", format="mp4",
|
| 182 |
webcam_options=gr.WebcamOptions())
|
| 183 |
vid_out = gr.Video(label="Annotated video")
|
| 184 |
plot_out = gr.Image(label="Fight probability curve")
|
| 185 |
beep_out = gr.Audio(label="Alarm")
|
| 186 |
-
gr.Button("Analyze").click(
|
| 187 |
-
|
|
|
|
|
|
|
|
|
|
| 188 |
|
| 189 |
-
#
|
| 190 |
with gr.Tab("Live stream"):
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
|
| 213 |
demo.queue().launch()
|
|
|
|
| 1 |
"""
|
| 2 |
+
app.py ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 3 |
+
Abnormal-Behaviour Detector
|
| 4 |
|
| 5 |
+
β’ Tab **Video file** β analyse an uploaded / recorded clip (your original
|
| 6 |
+
pipeline).
|
| 7 |
+
β’ Tab **Live stream** β real-time webcam analysis *plus* a rolling blue/red
|
| 8 |
+
bar-plot that updates every ~0.1 s.
|
| 9 |
+
|
| 10 |
+
No WebRTC here (plain Image streaming) so we can update multiple outputs at
|
| 11 |
+
once. Everything is self-contained; just drop the file in your Space and
|
| 12 |
+
keep the same `./model` folder.
|
| 13 |
"""
|
| 14 |
|
| 15 |
+
# ββ imports βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 16 |
+
import os, cv2, tempfile, numpy as np, matplotlib
|
| 17 |
+
matplotlib.use("Agg") # non-GUI backend for servers
|
| 18 |
+
import matplotlib.pyplot as plt
|
| 19 |
+
from collections import deque
|
| 20 |
|
| 21 |
import torch, gradio as gr, soundfile as sf
|
| 22 |
+
from transformers import VideoMAEForVideoClassification, AutoImageProcessor
|
| 23 |
+
|
| 24 |
+
# ββ model βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 25 |
+
MODEL_DIR = "./model" # path inside the repo
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 27 |
|
| 28 |
model = VideoMAEForVideoClassification.from_pretrained(MODEL_DIR).to(DEVICE)
|
| 29 |
processor = AutoImageProcessor.from_pretrained(MODEL_DIR, use_fast=False)
|
| 30 |
model.eval()
|
| 31 |
|
| 32 |
+
# ββ constants βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 33 |
+
CLIP = 16 # frames per clip (model was trained on 16)
|
| 34 |
+
STRIDE = 4 # evaluate every 4th frame
|
| 35 |
+
THRESH = 0.5 # fight/no-fight probability cut-off
|
| 36 |
+
|
| 37 |
+
# ββ batch-mode helpers (upload tab) βββββββββββββββββββββββββββββββββββββ
|
| 38 |
def sliding_window(path, clip_len=16, stride=4):
|
| 39 |
cap, frames = cv2.VideoCapture(path), []
|
| 40 |
fps = cap.get(cv2.CAP_PROP_FPS) or 25
|
|
|
|
| 47 |
|
| 48 |
probs, boxes = [], []
|
| 49 |
for s in range(0, len(frames) - clip_len, stride):
|
| 50 |
+
clip = [cv2.resize(frames[i], (224, 224))
|
| 51 |
+
for i in range(s, s + clip_len)]
|
| 52 |
inp = processor(clip, return_tensors="pt").to(DEVICE)
|
| 53 |
with torch.no_grad():
|
| 54 |
p = torch.softmax(model(**inp).logits, dim=1)[0, 1].item()
|
| 55 |
probs.append(p)
|
| 56 |
+
if p > THRESH:
|
| 57 |
boxes.append((s / fps, (s + clip_len) / fps))
|
| 58 |
return frames, probs, boxes, fps
|
| 59 |
|
| 60 |
+
def make_outputs(frames, probs, boxes, fps):
|
| 61 |
+
"""annotated video β’ probability plot β’ optional beep"""
|
| 62 |
h, w, _ = frames[0].shape
|
| 63 |
for t1, t2 in boxes:
|
| 64 |
for i in range(int(t1 * fps), min(int(t2 * fps), len(frames))):
|
| 65 |
cv2.rectangle(frames[i], (0, 0), (w - 1, h - 1), (255, 0, 0), 6)
|
| 66 |
|
| 67 |
+
# video
|
| 68 |
+
vid = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
|
| 69 |
+
vw = cv2.VideoWriter(vid, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h))
|
| 70 |
for f in frames:
|
| 71 |
vw.write(cv2.cvtColor(f, cv2.COLOR_RGB2BGR))
|
| 72 |
vw.release()
|
| 73 |
|
| 74 |
+
# plot
|
| 75 |
+
t = np.arange(len(probs)) * STRIDE / fps
|
| 76 |
+
fig, ax = plt.subplots(figsize=(5, 2))
|
| 77 |
+
ax.bar(t, probs,
|
| 78 |
+
color=["red" if p > THRESH else "blue" for p in probs],
|
| 79 |
+
width=STRIDE / fps)
|
| 80 |
+
ax.set_ylim(0, 1); ax.set_xlabel("sec"); ax.set_ylabel("fight prob")
|
| 81 |
+
fig.tight_layout()
|
| 82 |
+
plot = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
| 83 |
+
fig.savefig(plot, dpi=120)
|
| 84 |
+
plt.close(fig)
|
| 85 |
|
| 86 |
+
# beep
|
| 87 |
alarm = None
|
| 88 |
+
if any(p > THRESH for p in probs):
|
| 89 |
sr = 16_000
|
| 90 |
t_beep = np.linspace(0, 0.5, int(sr * 0.5), endpoint=False)
|
| 91 |
alarm = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
|
| 92 |
sf.write(alarm, 0.2 * np.sin(2 * np.pi * 440 * t_beep), sr)
|
| 93 |
|
| 94 |
+
return vid, plot, alarm
|
|
|
|
| 95 |
|
| 96 |
def run_pipeline(file_path):
|
| 97 |
if not file_path or not os.path.exists(file_path):
|
|
|
|
| 99 |
frames, probs, boxes, fps = sliding_window(file_path)
|
| 100 |
return make_outputs(frames, probs, boxes, fps)
|
| 101 |
|
| 102 |
+
# ββ live-stream helpers ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 103 |
+
def make_plot(probs):
|
| 104 |
+
"""return a Matplotlib figure (blue/red bars)"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
fig, ax = plt.subplots(figsize=(5, 2))
|
| 106 |
colors = ["red" if p > THRESH else "blue" for p in probs]
|
| 107 |
ax.bar(np.arange(len(probs)), probs, color=colors)
|
|
|
|
| 113 |
|
| 114 |
def live_handler(frame, state, probs):
|
| 115 |
"""
|
| 116 |
+
frame : RGB numpy array from webcam
|
| 117 |
+
state : deque of last 16 frames
|
| 118 |
+
probs : list of previous probabilities (len <= 60)
|
|
|
|
| 119 |
"""
|
| 120 |
state.append(frame)
|
| 121 |
+
if len(state) < CLIP: # warm-up stage, no inference yet
|
| 122 |
return state, frame, probs, make_plot(probs)
|
| 123 |
|
| 124 |
+
if live_handler.i % STRIDE == 0:
|
| 125 |
clip = [cv2.resize(f, (224, 224)) for f in list(state)]
|
| 126 |
inp = processor(clip, return_tensors="pt").to(DEVICE)
|
| 127 |
with torch.no_grad():
|
| 128 |
+
live_handler.p = torch.softmax(model(**inp).logits,
|
| 129 |
+
dim=1)[0, 1].item()
|
| 130 |
+
probs.append(live_handler.p)
|
| 131 |
+
if len(probs) > 60: # keep ~1 min of history
|
| 132 |
probs.pop(0)
|
| 133 |
live_handler.i += 1
|
| 134 |
|
| 135 |
+
# overlay
|
| 136 |
disp = frame.copy()
|
| 137 |
h, w = disp.shape[:2]
|
| 138 |
if live_handler.p > THRESH:
|
|
|
|
| 145 |
|
| 146 |
live_handler.i, live_handler.p = 0, 0.0
|
| 147 |
|
| 148 |
+
# ββ UI βββββββββββββββββββββββββββοΏ½οΏ½οΏ½βββββββββββββββββββββββββββββββββββββ
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
with gr.Blocks(title="Abnormal Behaviour Detection") as demo:
|
| 150 |
|
| 151 |
+
# βββ Tab 1 : video file / recorded upload ββββββββββββββββββββββββ
|
| 152 |
with gr.Tab("Video file"):
|
| 153 |
+
gr.Markdown(
|
| 154 |
+
"Upload a video **(.mp4 / .avi / .mov)** or record one, then click "
|
| 155 |
+
"**Analyze**."
|
| 156 |
+
)
|
| 157 |
file_in = gr.Video(label="Upload / record", format="mp4",
|
| 158 |
webcam_options=gr.WebcamOptions())
|
| 159 |
vid_out = gr.Video(label="Annotated video")
|
| 160 |
plot_out = gr.Image(label="Fight probability curve")
|
| 161 |
beep_out = gr.Audio(label="Alarm")
|
| 162 |
+
gr.Button("Analyze").click(
|
| 163 |
+
fn=run_pipeline,
|
| 164 |
+
inputs=file_in,
|
| 165 |
+
outputs=[vid_out, plot_out, beep_out],
|
| 166 |
+
)
|
| 167 |
|
| 168 |
+
# βββ Tab 2 : live webcam with rolling plot βββββββββββββββββββββββ
|
| 169 |
with gr.Tab("Live stream"):
|
| 170 |
+
gr.Markdown(
|
| 171 |
+
"Real-time fight detection. A **red border** appears around the "
|
| 172 |
+
"frame while a fight is detected; the bar plot turns **red** for "
|
| 173 |
+
"those clips and **blue** otherwise."
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
cam_in = gr.Image(
|
| 177 |
+
sources="webcam",
|
| 178 |
+
streaming=True,
|
| 179 |
+
type="numpy",
|
| 180 |
+
label="Webcam",
|
| 181 |
+
)
|
| 182 |
+
cam_out = gr.Image(streaming=True, label="Live annotated")
|
| 183 |
+
plot_live = gr.Plot(label="Fight probability (rolling)")
|
| 184 |
+
|
| 185 |
+
frame_state = gr.State(value=deque(maxlen=CLIP))
|
| 186 |
+
prob_state = gr.State(value=[])
|
| 187 |
+
|
| 188 |
+
cam_in.stream(
|
| 189 |
+
fn=live_handler,
|
| 190 |
+
inputs=[cam_in, frame_state, prob_state],
|
| 191 |
+
outputs=[frame_state, cam_out, prob_state, plot_live],
|
| 192 |
+
time_limit=0, # run indefinitely
|
| 193 |
+
stream_every=0.1 # backend tick (~10 Hz)
|
| 194 |
+
)
|
| 195 |
|
| 196 |
demo.queue().launch()
|