File size: 7,138 Bytes
d4753e9
5c4f87d
3f09490
 
 
 
73f7a9d
 
 
4592a71
d4753e9
 
f70b7cd
 
 
 
 
9135285
5c4f87d
0a46578
d4753e9
f70b7cd
 
 
0a46578
5c4f87d
73f7a9d
3f09490
25f8bf7
 
 
a865a06
681154d
73f7a9d
4592a71
 
 
 
 
 
 
32a939f
 
a865a06
32a939f
a865a06
 
32a939f
681154d
d4753e9
0a46578
3f09490
4592a71
 
 
f964e3a
f70b7cd
 
9135285
f964e3a
b730f2f
4592a71
4e66c8d
4592a71
 
 
 
 
73f7a9d
 
f964e3a
a865a06
 
4592a71
0a46578
4592a71
 
73f7a9d
 
5c4f87d
73f7a9d
 
4592a71
73f7a9d
4592a71
0a46578
73f7a9d
 
 
4592a71
73f7a9d
32a939f
b730f2f
73f7a9d
f70b7cd
a865a06
f70b7cd
73f7a9d
f964e3a
73f7a9d
 
 
 
f70b7cd
73f7a9d
f70b7cd
73f7a9d
 
b730f2f
cacd5f8
73f7a9d
d4753e9
4592a71
73f7a9d
3f09490
0a46578
4592a71
f70b7cd
 
 
3f09490
f70b7cd
 
d4753e9
73f7a9d
 
 
f70b7cd
 
 
4592a71
73f7a9d
d4753e9
4592a71
73f7a9d
 
 
f70b7cd
 
4592a71
73f7a9d
4592a71
73f7a9d
 
4592a71
 
73f7a9d
 
 
f70b7cd
 
4592a71
73f7a9d
f70b7cd
4592a71
f70b7cd
4592a71
 
73f7a9d
 
4592a71
 
73f7a9d
4592a71
f70b7cd
73f7a9d
 
f70b7cd
 
 
d4753e9
4592a71
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
"""
app.py  ──────────────────────────────────────────────────────────────────
Abnormal-Behaviour Detector (VideoMAE)

Tabs
────
1. **Video File**  – upload / record β†’ annotated video, static plot, optional beep.
2. **Live Stream** – real-time webcam with red-border overlay, rolling bar-plot, numeric prob.

**Fix:** ensure live always uses the full 16-frame clips the model was trained on.
"""

import os
import cv2
import tempfile
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from collections import deque

import torch
import gradio as gr
import soundfile as sf
from transformers import VideoMAEForVideoClassification, AutoImageProcessor

# ── Model Initialization ─────────────────────────────────────────────────
MODEL_DIR = "./model"
DEVICE    = "cuda" if torch.cuda.is_available() else "cpu"
model     = VideoMAEForVideoClassification.from_pretrained(MODEL_DIR).to(DEVICE)
processor = AutoImageProcessor.from_pretrained(MODEL_DIR, use_fast=False)
model.eval()

# ── Constants ─────────────────────────────────────────────────────────────
CLIP       = 16      # frames per clip used in both upload and live
STRIDE     = 1       # process every new frame in live mode
STREAM_FPS = 15      # polling rate for live (~15 updates/sec)
HIST_LEN   = 90      # rolling history length for bar plot

# ── Upload / Video-file pipeline ──────────────────────────────────────────
def sliding_window(path, clip_len=CLIP, stride=1):
    cap, frames = cv2.VideoCapture(path), []
    fps = cap.get(cv2.CAP_PROP_FPS) or 25
    while True:
        ok, f = cap.read()
        if not ok:
            break
        frames.append(cv2.cvtColor(f, cv2.COLOR_BGR2RGB))
    cap.release()

    probs = []
    for s in range(0, len(frames) - clip_len + 1, stride):
        clip = [cv2.resize(frames[i], (224,224))
                for i in range(s, s + clip_len)]
        inp = processor(clip, return_tensors="pt").to(DEVICE)
        with torch.no_grad():
            p = torch.softmax(model(**inp).logits, dim=1)[0,1].item()
        probs.append(p)
    return frames, probs, fps


def make_outputs(frames, probs, fps, sensitivity):
    h, w = frames[0].shape[:2]
    for idx, p in enumerate(probs):
        if p > sensitivity:
            for f in range(idx, min(idx + CLIP, len(frames))):
                cv2.rectangle(frames[f], (0,0), (w-1,h-1), (255,0,0), 6)

    out_vid = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False).name
    vw = cv2.VideoWriter(out_vid, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w,h))
    for f in frames:
        vw.write(cv2.cvtColor(f, cv2.COLOR_RGB2BGR))
    vw.release()

    t = np.arange(len(probs)) / fps
    fig, ax = plt.subplots(figsize=(5,2))
    colors = ['red' if p > sensitivity else 'blue' for p in probs]
    ax.bar(t, probs, color=colors, width=1/fps)
    ax.set_ylim(0,1); ax.set_xlabel('sec'); ax.set_ylabel('fight prob')
    fig.tight_layout()
    out_plot = tempfile.NamedTemporaryFile(suffix='.png', delete=False).name
    fig.savefig(out_plot, dpi=120); plt.close(fig)

    out_beep = None
    if any(p > sensitivity for p in probs):
        sr = 16000
        tone = 0.2 * np.sin(2 * np.pi * 440 * np.linspace(0,0.5,int(sr*0.5)))
        out_beep = tempfile.NamedTemporaryFile(suffix='.wav', delete=False).name
        sf.write(out_beep, tone, sr)

    return out_vid, out_plot, out_beep


def run_pipeline(video_path, sensitivity):
    if not video_path or not os.path.exists(video_path):
        return None, None, None
    frames, probs, fps = sliding_window(video_path)
    return make_outputs(frames, probs, fps, sensitivity)

# ── Live-stream pipeline ─────────────────────────────────────────────────
def plot_history(history, sensitivity):
    fig, ax = plt.subplots(figsize=(5,2))
    colors = ['red' if p > sensitivity else 'blue' for p in history]
    ax.bar(np.arange(len(history)), history, color=colors)
    ax.set_ylim(0,1)
    ax.set_xlabel('recent clips'); ax.set_ylabel('fight prob')
    fig.tight_layout()
    return fig


def live_handler(frame, state, history, sensitivity):
    state.append(frame)
    if len(state) < CLIP:
        return state, frame, history, plot_history(history, sensitivity), 0.0

    clip = [cv2.resize(f, (224,224)) for f in list(state)]
    inp = processor(clip, return_tensors='pt').to(DEVICE)
    with torch.no_grad():
        p = torch.softmax(model(**inp).logits, dim=1)[0,1].item()

    history.append(p)
    if len(history) > HIST_LEN:
        history.pop(0)

    disp = frame.copy()
    h, w = disp.shape[:2]
    if p > sensitivity:
        cv2.rectangle(disp, (0,0), (w-1,h-1), (255,0,0), 6)
    cv2.putText(disp, f'fight prob: {p:.2f}', (10, h-15),
                cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255), 2, cv2.LINE_AA)

    return state, disp, history, plot_history(history, sensitivity), p

# ── Gradio UI ───────────────────────────────────────────────────────────
with gr.Blocks(title="Abnormal Behaviour Detection") as demo:
    sensitivity = gr.Slider(
        label="Sensitivity threshold (lower β†’ more sensitive)",
        minimum=0.0, maximum=1.0, step=0.01, value=0.0
    )

    with gr.Tab("Video File"):
        gr.Markdown("Upload or record a video, set sensitivity, then Analyze.")
        file_in  = gr.Video(format='mp4', webcam_options=gr.WebcamOptions())
        vid_out  = gr.Video(label='Annotated Video')
        plot_out = gr.Image(label='Fight Probability Curve')
        beep_out = gr.Audio(label='Alarm Sound')
        gr.Button("Analyze").click(
            fn=run_pipeline,
            inputs=[file_in, sensitivity],
            outputs=[vid_out, plot_out, beep_out]
        )

    with gr.Tab("Live Stream"):
        gr.Markdown(
            "Live webcam. Border & plot turn red when `prob > sensitivity`."
        )
        cam_in  = gr.Image(sources='webcam', streaming=True, type='numpy', label='Webcam')
        cam_out = gr.Image(streaming=True, label='Annotated Feed')
        plot_live = gr.Plot(label='Rolling Fight Probability')
        num_prob  = gr.Number(label='Current Probability', precision=2)

        frame_state = gr.State(value=deque(maxlen=CLIP))
        prob_state  = gr.State(value=[])

        cam_in.stream(
            fn=live_handler,
            inputs=[cam_in, frame_state, prob_state, sensitivity],
            outputs=[frame_state, cam_out, prob_state, plot_live, num_prob],
            stream_every=1/STREAM_FPS
        )

demo.queue().launch()