File size: 7,078 Bytes
e5c09aa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
import json
import os
from typing import Dict, List, Tuple, Optional, Iterable
from tqdm import tqdm

# ================= Configuration =================
PRED_PATH = "result.jsonl"

GT_PATH = "gt.jsonl"

THRESHOLDS = [0.7, 0.8, 0.9]

WINDOW_SIZE = 1 
# ===========================================================

def load_jsonl(path: str) -> List[dict]:
    data = []
    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line: 
                continue
            try:
                data.append(json.loads(line))
            except json.JSONDecodeError:
                pass
    return data

def load_json_or_jsonl(path: str) -> List[dict]:
    """Support both JSON and JSONL formats"""
    with open(path, "r", encoding="utf-8") as f:
        first_char = f.read(1)
        f.seek(0)
        if first_char == '[':
            return json.load(f)
        else:
            return load_jsonl(path)

def build_gt_index(gt_items: Iterable[dict]) -> Dict[str, Tuple[int, int]]:
    """Build {video_id: (start, end)} mapping"""
    gt = {}
    for item in gt_items:
        vid = str(item.get("id"))
        
        # Adapt to different GT structures
        seg = None
        ans = item.get("answer", [])
        if ans:
            # Try to obtain the segment field
            seg_field = ans.get("segment")  # Could be [start, end] or [[s,e]]
            if isinstance(seg_field, list) and len(seg_field) > 0:
                if isinstance(seg_field[0], list):  # Nested list [[s, e]]
                    s, e = seg_field[0]
                elif len(seg_field) == 2:  # Direct list [s, e]
                    s, e = seg_field
                else:
                    continue
                seg = (int(s), int(e))
        
        if seg:
            gt[vid] = seg
    return gt

def get_trigger_events(probs: List[float], threshold: float, window: int) -> List[int]:
    """

    Return all trigger timestamps (indices).

    Logic: must have window consecutive frames >= threshold.

    The trigger timestamp is defined as the last frame of the window.

    """
    triggers = []
    
    # If window is 1, directly check
    if window <= 1:
        return [i for i, p in enumerate(probs) if p >= threshold]
    
    # Window-based logic
    if len(probs) < window:
        return []

    for i in range(len(probs) - window + 1):
        chunk = probs[i : i + window]
        if all(p >= threshold for p in chunk):
            triggers.append(i + window - 1)
            
    return triggers

def count_distinct_groups(indices: List[int]) -> int:
    """

    Count how many independent trigger groups exist.

    Example: indices = [10, 11, 12, 50, 51]

    [10,11,12] is group 1 (continuous)

    [50,51] is group 2

    Return 2. This better reflects how "verbose" the model is.

    """
    if not indices:
        return 0
        
    groups = 1
    for i in range(1, len(indices)):
        # If the difference between indices is greater than 1, a new group begins
        if indices[i] - indices[i-1] > 1:
            groups += 1
    return groups

def main():
    print(f"Loading predictions from: {os.path.basename(PRED_PATH)}")
    preds = load_jsonl(PRED_PATH)
    
    print(f"Loading Ground Truth from: {os.path.basename(GT_PATH)}")
    gt_raw = load_json_or_jsonl(GT_PATH)
    gt_index = build_gt_index(gt_raw)
    
    print(f"Total Preds: {len(preds)} | Total GT: {len(gt_index)}")
    
    stats = {thr: {
        'total': 0,
        'correct': 0,
        'early': 0,
        'late': 0,
        'miss': 0,
        'repeated_count': 0  # number of samples with multiple independent alarms
    } for thr in THRESHOLDS}
    
    missing_gt_count = 0
    
    for item in tqdm(preds, desc="Analyzing"):
        vid = str(item.get("id"))
        probs = item.get("raw_probs", [])
        
        if vid not in gt_index:
            missing_gt_count += 1
            continue
            
        start, end = gt_index[vid]
        
        for thr in THRESHOLDS:
            stats[thr]['total'] += 1
            
            # 1. Get all trigger points
            trigger_indices = get_trigger_events(probs, thr, WINDOW_SIZE)
            
            if not trigger_indices:
                # No trigger during the entire sequence -> Miss
                stats[thr]['miss'] += 1
                continue
                
            # 2. Get the first trigger time
            first_idx = trigger_indices[0]
            
            # 3. Determine Early / Late / Correct
            # If the first trigger lies within the interval, it counts as Correct
            if first_idx < start:
                stats[thr]['early'] += 1
            elif first_idx > end:
                stats[thr]['late'] += 1
            else:
                stats[thr]['correct'] += 1
                
            # 4. Determine Repeated Triggers (whether the model is verbose)
            # Count how many independent alarm segments exist
            num_groups = count_distinct_groups(trigger_indices)
            if num_groups > 1:
                stats[thr]['repeated_count'] += 1

    # ================= Output Report =================
    print("\n" + "="*95)
    print(f"TRIGGER ERROR ANALYSIS (Window Size = {WINDOW_SIZE})")
    print(f"Valid Samples: {stats[THRESHOLDS[0]]['total']} | Missing GT: {missing_gt_count}")
    print("="*95)
    
    # Table header
    headers = ["Thr", "Correct(Hit)", "Early", "Late", "Missed", "FP(E+L)", "Repeated%"]
    row_fmt = "{:<6} | {:<12} | {:<10} | {:<10} | {:<10} | {:<10} | {:<10}"
    
    print(row_fmt.format(*headers))
    print("-" * 95)
    
    for thr in THRESHOLDS:
        s = stats[thr]
        total = s['total']
        if total == 0: 
            continue
        
        # Calculate percentages
        p_corr = s['correct'] / total * 100
        p_early = s['early'] / total * 100
        p_late = s['late'] / total * 100
        p_miss = s['miss'] / total * 100
        p_fp = p_early + p_late  # False Positive = Early + Late
        p_rep = s['repeated_count'] / total * 100
        
        print(row_fmt.format(
            f"{thr}",
            f"{p_corr:.1f}%",
            f"{p_early:.1f}%",
            f"{p_late:.1f}%",
            f"{p_miss:.1f}%",
            f"{p_fp:.1f}%",
            f"{p_rep:.1f}%"
        ))
    
    print("="*95)
    print("Metrics Definition:")
    print("1. Correct: First trigger occurs inside [Start, End].")
    print("2. Early:   First trigger occurs before Start.")
    print("3. Late:    First trigger occurs after End.")
    print("4. FP:      False Positives (Early + Late).")
    print("5. Repeated%: Percentage of videos that triggered distinct alarms more than once.")
    print("              (e.g., alarm at 10s... stop... alarm again at 50s).")

if __name__ == "__main__":
    main()