Myteam commited on
Commit
5c4f87d
Β·
verified Β·
1 Parent(s): cacd5f8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +97 -114
app.py CHANGED
@@ -1,37 +1,40 @@
1
  """
2
- app.py – Abnormal-Behaviour Detector
3
- (upload / recorded video + live-webcam stream)
4
 
5
- Fix: remove State from WebRTC.stream() outputs.
 
 
 
 
 
 
 
6
  """
7
 
8
- # ───────────────────────────── imports ──────────────────────────────
9
- import os, cv2, tempfile, numpy as np, matplotlib.pyplot as plt
10
- from collections import deque
 
 
11
 
12
  import torch, gradio as gr, soundfile as sf
13
- from transformers import VideoMAEForVideoClassification, AutoImageProcessor
14
-
15
- # try FastRTC first (new package), else gradio-webrtc
16
- try:
17
- from fastrtc import WebRTC
18
- webrtc_ok = True
19
- except ModuleNotFoundError:
20
- try:
21
- from gradio_webrtc import WebRTC
22
- webrtc_ok = True
23
- except ModuleNotFoundError:
24
- webrtc_ok = False
25
-
26
- # ─────────────────────────── model load ────────────────────────────
27
- MODEL_DIR = "./model"
28
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
29
 
30
  model = VideoMAEForVideoClassification.from_pretrained(MODEL_DIR).to(DEVICE)
31
  processor = AutoImageProcessor.from_pretrained(MODEL_DIR, use_fast=False)
32
  model.eval()
33
 
34
- # ───────────────────── batch-video helpers (unchanged) ─────────────
 
 
 
 
 
35
  def sliding_window(path, clip_len=16, stride=4):
36
  cap, frames = cv2.VideoCapture(path), []
37
  fps = cap.get(cv2.CAP_PROP_FPS) or 25
@@ -44,48 +47,51 @@ def sliding_window(path, clip_len=16, stride=4):
44
 
45
  probs, boxes = [], []
46
  for s in range(0, len(frames) - clip_len, stride):
47
- clip = [cv2.resize(frames[i], (224, 224)) for i in range(s, s + clip_len)]
 
48
  inp = processor(clip, return_tensors="pt").to(DEVICE)
49
  with torch.no_grad():
50
  p = torch.softmax(model(**inp).logits, dim=1)[0, 1].item()
51
  probs.append(p)
52
- if p > 0.5:
53
  boxes.append((s / fps, (s + clip_len) / fps))
54
  return frames, probs, boxes, fps
55
 
56
-
57
- def make_outputs(frames, probs, boxes, fps, thresh=0.5):
58
  h, w, _ = frames[0].shape
59
  for t1, t2 in boxes:
60
  for i in range(int(t1 * fps), min(int(t2 * fps), len(frames))):
61
  cv2.rectangle(frames[i], (0, 0), (w - 1, h - 1), (255, 0, 0), 6)
62
 
63
- vid_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
64
- vw = cv2.VideoWriter(vid_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h))
 
65
  for f in frames:
66
  vw.write(cv2.cvtColor(f, cv2.COLOR_RGB2BGR))
67
  vw.release()
68
 
69
- t = np.arange(len(probs)) * 4 / fps
70
- plt.figure(figsize=(5, 2))
71
- plt.bar(t, probs, width=4 / fps,
72
- color=["red" if p > thresh else "blue" for p in probs])
73
- plt.ylim(0, 1)
74
- plt.xlabel("sec")
75
- plt.ylabel("fight prob")
76
- plot_path = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
77
- plt.savefig(plot_path, bbox_inches="tight")
78
- plt.close()
 
79
 
 
80
  alarm = None
81
- if any(p > thresh for p in probs):
82
  sr = 16_000
83
  t_beep = np.linspace(0, 0.5, int(sr * 0.5), endpoint=False)
84
  alarm = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
85
  sf.write(alarm, 0.2 * np.sin(2 * np.pi * 440 * t_beep), sr)
86
 
87
- return vid_path, plot_path, alarm
88
-
89
 
90
  def run_pipeline(file_path):
91
  if not file_path or not os.path.exists(file_path):
@@ -93,13 +99,9 @@ def run_pipeline(file_path):
93
  frames, probs, boxes, fps = sliding_window(file_path)
94
  return make_outputs(frames, probs, boxes, fps)
95
 
96
- # ────────────────────── live-stream handlers ───────────────────────
97
- CLIP, STRIDE, THRESH = 16, 4, 0.5
98
-
99
- def make_plot(probs): # helper to draw the bar plot
100
- import matplotlib.pyplot as plt
101
- import numpy as np
102
-
103
  fig, ax = plt.subplots(figsize=(5, 2))
104
  colors = ["red" if p > THRESH else "blue" for p in probs]
105
  ax.bar(np.arange(len(probs)), probs, color=colors)
@@ -111,27 +113,26 @@ def make_plot(probs): # helper to draw the bar plot
111
 
112
  def live_handler(frame, state, probs):
113
  """
114
- frame : numpy RGB image from webcam
115
- state : deque holding last 16 frames
116
- probs : list of past probabilities (for the plot)
117
- returns: updated state, annotated frame, updated probs, plot-figure
118
  """
119
  state.append(frame)
120
- if len(state) < CLIP: # warm-up
121
  return state, frame, probs, make_plot(probs)
122
 
123
- if live_handler.i % STRIDE == 0: # run model every 4th frame
124
  clip = [cv2.resize(f, (224, 224)) for f in list(state)]
125
  inp = processor(clip, return_tensors="pt").to(DEVICE)
126
  with torch.no_grad():
127
- p = torch.softmax(model(**inp).logits, dim=1)[0, 1].item()
128
- live_handler.p = p
129
- probs.append(p)
130
- if len(probs) > 60: # keep ~1 min history (60 clips)
131
  probs.pop(0)
132
  live_handler.i += 1
133
 
134
- # ── overlay ──
135
  disp = frame.copy()
136
  h, w = disp.shape[:2]
137
  if live_handler.p > THRESH:
@@ -144,70 +145,52 @@ def live_handler(frame, state, probs):
144
 
145
  live_handler.i, live_handler.p = 0, 0.0
146
 
147
- # -------- WebRTC path uses a global deque ---------------------------
148
- buffer_webrtc = deque(maxlen=CLIP)
149
-
150
- def live_handler_webrtc(frame):
151
- buffer_webrtc.append(frame)
152
- if len(buffer_webrtc) < CLIP:
153
- return frame
154
-
155
- if live_handler_webrtc.i % STRIDE == 0:
156
- clip = [cv2.resize(f, (224, 224)) for f in list(buffer_webrtc)]
157
- inp = processor(clip, return_tensors="pt").to(DEVICE)
158
- with torch.no_grad():
159
- live_handler_webrtc.p = torch.softmax(
160
- model(**inp).logits, dim=1
161
- )[0, 1].item()
162
- live_handler_webrtc.i += 1
163
-
164
- disp = frame.copy()
165
- h, w = disp.shape[:2]
166
- if live_handler_wertc.p > THRESH:
167
- cv2.rectangle(disp, (0, 0), (w - 1, h - 1), (255, 0, 0), 6)
168
- cv2.putText(disp, f"fight prob: {live_handler_webrtc.p:.2f}",
169
- (10, h - 15), cv2.FONT_HERSHEY_SIMPLEX, 0.7,
170
- (255, 255, 255), 2, cv2.LINE_AA)
171
- return disp
172
-
173
- live_handler_webrtc.i, live_handler_webrtc.p = 0, 0.0
174
-
175
- # ───────────────────────────── UI ───────────────────────────────────
176
  with gr.Blocks(title="Abnormal Behaviour Detection") as demo:
177
 
178
- # ---- TAB 1 : upload / recorded video ---------------------------
179
  with gr.Tab("Video file"):
180
- gr.Markdown("Upload a video **(.mp4 / .avi / .mov)** or record one.")
 
 
 
181
  file_in = gr.Video(label="Upload / record", format="mp4",
182
  webcam_options=gr.WebcamOptions())
183
  vid_out = gr.Video(label="Annotated video")
184
  plot_out = gr.Image(label="Fight probability curve")
185
  beep_out = gr.Audio(label="Alarm")
186
- gr.Button("Analyze").click(run_pipeline, file_in,
187
- [vid_out, plot_out, beep_out])
 
 
 
188
 
189
- # ---- TAB 2 : live webcam --------------------------------------
190
  with gr.Tab("Live stream"):
191
- gr.Markdown(
192
- "Real-time fight detection. A **red border** appears while a fight is "
193
- "detected; the bar plot below turns **red** for those clips."
194
- )
195
-
196
- cam_in = gr.Image(sources="webcam", streaming=True,
197
- type="numpy", label="Webcam")
198
- cam_out = gr.Image(streaming=True, label="Live annotated")
199
- plot_out = gr.Plot(label="Fight probability (rolling)")
200
-
201
- # shared states
202
- frame_state = gr.State(value=deque(maxlen=CLIP))
203
- prob_state = gr.State(value=[])
204
-
205
- cam_in.stream(
206
- fn=live_handler,
207
- inputs=[cam_in, frame_state, prob_state],
208
- outputs=[frame_state, cam_out, prob_state, plot_out],
209
- time_limit=0, # run indefinitely
210
- stream_every=0.1 # ~10 Hz backend calls
211
- )
 
 
 
 
212
 
213
  demo.queue().launch()
 
1
  """
2
+ app.py ──────────────────────────────────────────────────────────────────
3
+ Abnormal-Behaviour Detector
4
 
5
+ β€’ Tab **Video file** – analyse an uploaded / recorded clip (your original
6
+ pipeline).
7
+ β€’ Tab **Live stream** – real-time webcam analysis *plus* a rolling blue/red
8
+ bar-plot that updates every ~0.1 s.
9
+
10
+ No WebRTC here (plain Image streaming) so we can update multiple outputs at
11
+ once. Everything is self-contained; just drop the file in your Space and
12
+ keep the same `./model` folder.
13
  """
14
 
15
+ # ── imports ─────────────────────────────────────────────────────────────
16
+ import os, cv2, tempfile, numpy as np, matplotlib
17
+ matplotlib.use("Agg") # non-GUI backend for servers
18
+ import matplotlib.pyplot as plt
19
+ from collections import deque
20
 
21
  import torch, gradio as gr, soundfile as sf
22
+ from transformers import VideoMAEForVideoClassification, AutoImageProcessor
23
+
24
+ # ── model ───────────────────────────────────────────────────────────────
25
+ MODEL_DIR = "./model" # path inside the repo
 
 
 
 
 
 
 
 
 
 
 
26
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
27
 
28
  model = VideoMAEForVideoClassification.from_pretrained(MODEL_DIR).to(DEVICE)
29
  processor = AutoImageProcessor.from_pretrained(MODEL_DIR, use_fast=False)
30
  model.eval()
31
 
32
+ # ── constants ───────────────────────────────────────────────────────────
33
+ CLIP = 16 # frames per clip (model was trained on 16)
34
+ STRIDE = 4 # evaluate every 4th frame
35
+ THRESH = 0.5 # fight/no-fight probability cut-off
36
+
37
+ # ── batch-mode helpers (upload tab) ─────────────────────────────────────
38
  def sliding_window(path, clip_len=16, stride=4):
39
  cap, frames = cv2.VideoCapture(path), []
40
  fps = cap.get(cv2.CAP_PROP_FPS) or 25
 
47
 
48
  probs, boxes = [], []
49
  for s in range(0, len(frames) - clip_len, stride):
50
+ clip = [cv2.resize(frames[i], (224, 224))
51
+ for i in range(s, s + clip_len)]
52
  inp = processor(clip, return_tensors="pt").to(DEVICE)
53
  with torch.no_grad():
54
  p = torch.softmax(model(**inp).logits, dim=1)[0, 1].item()
55
  probs.append(p)
56
+ if p > THRESH:
57
  boxes.append((s / fps, (s + clip_len) / fps))
58
  return frames, probs, boxes, fps
59
 
60
+ def make_outputs(frames, probs, boxes, fps):
61
+ """annotated video β€’ probability plot β€’ optional beep"""
62
  h, w, _ = frames[0].shape
63
  for t1, t2 in boxes:
64
  for i in range(int(t1 * fps), min(int(t2 * fps), len(frames))):
65
  cv2.rectangle(frames[i], (0, 0), (w - 1, h - 1), (255, 0, 0), 6)
66
 
67
+ # video
68
+ vid = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
69
+ vw = cv2.VideoWriter(vid, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h))
70
  for f in frames:
71
  vw.write(cv2.cvtColor(f, cv2.COLOR_RGB2BGR))
72
  vw.release()
73
 
74
+ # plot
75
+ t = np.arange(len(probs)) * STRIDE / fps
76
+ fig, ax = plt.subplots(figsize=(5, 2))
77
+ ax.bar(t, probs,
78
+ color=["red" if p > THRESH else "blue" for p in probs],
79
+ width=STRIDE / fps)
80
+ ax.set_ylim(0, 1); ax.set_xlabel("sec"); ax.set_ylabel("fight prob")
81
+ fig.tight_layout()
82
+ plot = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
83
+ fig.savefig(plot, dpi=120)
84
+ plt.close(fig)
85
 
86
+ # beep
87
  alarm = None
88
+ if any(p > THRESH for p in probs):
89
  sr = 16_000
90
  t_beep = np.linspace(0, 0.5, int(sr * 0.5), endpoint=False)
91
  alarm = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
92
  sf.write(alarm, 0.2 * np.sin(2 * np.pi * 440 * t_beep), sr)
93
 
94
+ return vid, plot, alarm
 
95
 
96
  def run_pipeline(file_path):
97
  if not file_path or not os.path.exists(file_path):
 
99
  frames, probs, boxes, fps = sliding_window(file_path)
100
  return make_outputs(frames, probs, boxes, fps)
101
 
102
+ # ── live-stream helpers ────────────────────────────────────────────────
103
+ def make_plot(probs):
104
+ """return a Matplotlib figure (blue/red bars)"""
 
 
 
 
105
  fig, ax = plt.subplots(figsize=(5, 2))
106
  colors = ["red" if p > THRESH else "blue" for p in probs]
107
  ax.bar(np.arange(len(probs)), probs, color=colors)
 
113
 
114
  def live_handler(frame, state, probs):
115
  """
116
+ frame : RGB numpy array from webcam
117
+ state : deque of last 16 frames
118
+ probs : list of previous probabilities (len <= 60)
 
119
  """
120
  state.append(frame)
121
+ if len(state) < CLIP: # warm-up stage, no inference yet
122
  return state, frame, probs, make_plot(probs)
123
 
124
+ if live_handler.i % STRIDE == 0:
125
  clip = [cv2.resize(f, (224, 224)) for f in list(state)]
126
  inp = processor(clip, return_tensors="pt").to(DEVICE)
127
  with torch.no_grad():
128
+ live_handler.p = torch.softmax(model(**inp).logits,
129
+ dim=1)[0, 1].item()
130
+ probs.append(live_handler.p)
131
+ if len(probs) > 60: # keep ~1 min of history
132
  probs.pop(0)
133
  live_handler.i += 1
134
 
135
+ # overlay
136
  disp = frame.copy()
137
  h, w = disp.shape[:2]
138
  if live_handler.p > THRESH:
 
145
 
146
  live_handler.i, live_handler.p = 0, 0.0
147
 
148
+ # ── UI ───────────────────────────���─────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  with gr.Blocks(title="Abnormal Behaviour Detection") as demo:
150
 
151
+ # ─── Tab 1 : video file / recorded upload ────────────────────────
152
  with gr.Tab("Video file"):
153
+ gr.Markdown(
154
+ "Upload a video **(.mp4 / .avi / .mov)** or record one, then click "
155
+ "**Analyze**."
156
+ )
157
  file_in = gr.Video(label="Upload / record", format="mp4",
158
  webcam_options=gr.WebcamOptions())
159
  vid_out = gr.Video(label="Annotated video")
160
  plot_out = gr.Image(label="Fight probability curve")
161
  beep_out = gr.Audio(label="Alarm")
162
+ gr.Button("Analyze").click(
163
+ fn=run_pipeline,
164
+ inputs=file_in,
165
+ outputs=[vid_out, plot_out, beep_out],
166
+ )
167
 
168
+ # ─── Tab 2 : live webcam with rolling plot ───────────────────────
169
  with gr.Tab("Live stream"):
170
+ gr.Markdown(
171
+ "Real-time fight detection. A **red border** appears around the "
172
+ "frame while a fight is detected; the bar plot turns **red** for "
173
+ "those clips and **blue** otherwise."
174
+ )
175
+
176
+ cam_in = gr.Image(
177
+ sources="webcam",
178
+ streaming=True,
179
+ type="numpy",
180
+ label="Webcam",
181
+ )
182
+ cam_out = gr.Image(streaming=True, label="Live annotated")
183
+ plot_live = gr.Plot(label="Fight probability (rolling)")
184
+
185
+ frame_state = gr.State(value=deque(maxlen=CLIP))
186
+ prob_state = gr.State(value=[])
187
+
188
+ cam_in.stream(
189
+ fn=live_handler,
190
+ inputs=[cam_in, frame_state, prob_state],
191
+ outputs=[frame_state, cam_out, prob_state, plot_live],
192
+ time_limit=0, # run indefinitely
193
+ stream_every=0.1 # backend tick (~10 Hz)
194
+ )
195
 
196
  demo.queue().launch()