Zaafan commited on
Commit
ed24203
·
verified ·
1 Parent(s): 92843c8

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +117 -2011
README.md CHANGED
@@ -1,2056 +1,162 @@
1
- """
2
- CV Inference Service (Main Entry Point)
3
- =======================================================
4
- Orchestrates the full detection → tracking → motion → activity pipeline.
5
- Produces equipment telemetry events to Kafka.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
- Pipeline per frame:
8
- 1. Video Ingestion → FramePacket
9
- 2. (Optional) Zero-DCE low-light enhancement
10
- 3. RF-DETR Detection → raw detections [x1,y1,x2,y2,conf,cls]
11
- 4. TAI Heuristic Filter → clean detections (suppress fragments)
12
- 5. BoT-SORT Tracking → tracked objects with persistent IDs
13
- 6. DINOv3 Re-ID Gallery → re-identify lost tracks
14
- 7. Grid Decomposition + Farneback → articulated motion detection
15
- 8. Rule-based Activity Classifier → DIGGING/LOADING/DUMPING/WAITING
16
- 9. Kafka Producer → equipment.telemetry.raw
17
- """
18
 
19
- import os
20
- import sys
21
- import json
22
- import time
23
- import logging
24
- import signal
25
- from datetime import datetime, timezone
26
- from pathlib import Path
27
 
28
- import cv2
29
- import numpy as np
30
- import torch
 
31
 
 
32
 
33
- # ── Model weight resolver ────────────────────────────────────────────────
34
- HF_REPO_ID = "Zaafan/sitesense-weights"
35
 
36
- def resolve_weights(filename: str, local_dir: str = "/models") -> str:
37
- """Return the path to a model weight file.
38
-
39
- Priority:
40
- 1. Local file at <local_dir>/<filename> (fastest — no network)
41
- 2. Auto-download from Hugging Face Hub (fallback)
42
- """
43
- local_path = os.path.join(local_dir, filename)
44
- if os.path.exists(local_path):
45
- logger.info(f"Found local weights: {local_path}")
46
- return local_path
47
 
48
- logger.warning(f"Weights not found at {local_path} — downloading from HuggingFace ({HF_REPO_ID})...")
49
- try:
50
- from huggingface_hub import hf_hub_download
51
- downloaded = hf_hub_download(
52
- repo_id=HF_REPO_ID,
53
- filename=filename,
54
- local_dir=local_dir,
55
- )
56
- logger.info(f"Downloaded weights to: {downloaded}")
57
- return downloaded
58
- except Exception as e:
59
- logger.error(f"Failed to download {filename} from HuggingFace: {e}")
60
- raise FileNotFoundError(
61
- f"Model weights '{filename}' not found locally at {local_path} "
62
- f"and could not be downloaded from {HF_REPO_ID}. "
63
- f"Run: huggingface-cli download {HF_REPO_ID} --local-dir {local_dir}"
64
- )
65
- from collections import deque
66
- from PIL import Image # Moved to top level (audit #11)
67
 
68
- # Add parent directories to path for imports
69
- sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
70
- sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'video-ingestion'))
71
 
72
- logging.basicConfig(
73
- level=logging.INFO,
74
- format='%(asctime)s [%(name)s] %(levelname)s: %(message)s'
75
- )
76
- logger = logging.getLogger('cv-inference')
77
 
78
- # Class mapping matching our trained model (8-class v3)
79
- CLASS_NAMES = {
80
- 0: 'excavator', 1: 'dump_truck', 2: 'bulldozer', 3: 'wheel_loader',
81
- 4: 'mobile_crane', 5: 'tower_crane', 6: 'roller_compactor', 7: 'cement_mixer',
82
- }
83
- CLASS_PREFIXES = {
84
- 0: 'EX', 1: 'DT', 2: 'BD', 3: 'WL',
85
- 4: 'MC', 5: 'TC', 6: 'RC', 7: 'CM',
86
- }
87
-
88
-
89
- # =============================================================================
90
- # RF-DETR Detector Wrapper
91
- # =============================================================================
92
- class RFDETRDetector:
93
- """
94
- Wraps the fine-tuned RF-DETR model for inference.
95
- Loads checkpoint_best_regular.pth trained on our 8-class dataset.
96
- """
97
-
98
- def __init__(self, weights_path: str, num_classes: int = 8,
99
- confidence_threshold: float = 0.35, device: str = 'cuda'):
100
- self.confidence_threshold = confidence_threshold
101
- self.device = device if torch.cuda.is_available() else 'cpu'
102
- self.model = None
103
- self.weights_path = weights_path
104
-
105
- def load(self):
106
- """Load the fine-tuned RF-DETR model."""
107
- from rfdetr import RFDETRBase
108
- # In rfdetr, the weights path is passed as `pretrain_weights`
109
- self.model = RFDETRBase(num_classes=8, pretrain_weights=self.weights_path)
110
- # Optimize model for faster inference (torch.compile / fused ops)
111
- if hasattr(self.model, 'optimize_for_inference'):
112
- self.model.optimize_for_inference()
113
- logger.info("RF-DETR inference optimization applied")
114
- logger.info(f"RF-DETR loaded from {self.weights_path} on {self.device}")
115
-
116
- def predict(self, frame: np.ndarray) -> np.ndarray:
117
- """
118
- Run detection on a single frame.
119
-
120
- Args:
121
- frame: BGR image (H, W, 3)
122
-
123
- Returns:
124
- (N, 6) array of [x1, y1, x2, y2, confidence, class_id]
125
- """
126
- if self.model is None:
127
- return np.empty((0, 6))
128
-
129
- detections = self.model.predict(frame, threshold=self.confidence_threshold)
130
-
131
- # RF-DETR returns a Supervision Detections object
132
- # with .xyxy (N,4), .confidence (N,), .class_id (N,)
133
- if detections is None or len(detections) == 0:
134
- return np.empty((0, 6))
135
-
136
- boxes = detections.xyxy # (N, 4)
137
- confs = detections.confidence.reshape(-1, 1) # (N, 1)
138
- clses = detections.class_id.reshape(-1, 1) # (N, 1)
139
-
140
- return np.hstack([boxes, confs, clses]).astype(np.float32)
141
-
142
-
143
- # =============================================================================
144
- # DINOv3 Re-ID Gallery (Persistent Identity Across Occlusions)
145
- # =============================================================================
146
- class DINOv3ReIDGallery:
147
- """
148
- Uses DINOv3 ViT-B/16 to extract visual embeddings from equipment crops.
149
- Maintains a gallery of lost tracks to re-identify them when they reappear.
150
-
151
- This solves the critical problem: when two identical yellow haul trucks
152
- drive behind a dirt mound, the tracker loses their IDs. This gallery
153
- matches them back to their original identity using visual fingerprints.
154
- """
155
-
156
- def __init__(self, similarity_threshold: float = 0.95,
157
- gallery_ttl_frames: int = 1800, device: str = 'cuda',
158
- color_threshold: float = 0.40,
159
- spatial_gate_ratio: float = 0.40):
160
- self.color_threshold = color_threshold
161
- self.spatial_gate_ratio = spatial_gate_ratio
162
- self.similarity_threshold = similarity_threshold
163
- self.gallery_ttl_frames = gallery_ttl_frames
164
- self.device = device if torch.cuda.is_available() else 'cpu'
165
-
166
- self.model = None
167
- self.processor = None
168
- self.projection = None
169
-
170
- # Gallery: track_id -> {embedding, equipment_id, class_id, last_frame, bbox}
171
- self._gallery = {}
172
- # Active tracks: track_id -> embedding (updated every N frames)
173
- self._active_embeddings = {}
174
-
175
- def load(self):
176
- """Load DINOv3 ViT-B/16 from HuggingFace."""
177
- from transformers import AutoImageProcessor, AutoModel
178
-
179
- # Path to the mounted container volume pointing to P:\Projects\Eagle Vision Project\models\dinov3-vitb16-pretrain-lvd1689m
180
- model_id = "/models/dinov3-vitb16-pretrain-lvd1689m"
181
- logger.info(f"Loading DINOv3 Re-ID model locally from: {model_id}")
182
-
183
- self.processor = AutoImageProcessor.from_pretrained(model_id)
184
- self.model = AutoModel.from_pretrained(model_id).to(self.device)
185
- self.model.eval()
186
-
187
- # Load contrastive-trained projection head if available
188
- # Architecture must match training/train_reid.py
189
- import torch.nn as nn
190
- self.projection = nn.Sequential(
191
- nn.Linear(1536, 768),
192
- nn.BatchNorm1d(768),
193
- nn.GELU(),
194
- nn.Dropout(0.1),
195
- nn.Linear(768, 256),
196
- nn.BatchNorm1d(256),
197
- nn.GELU(),
198
- nn.Dropout(0.1),
199
- nn.Linear(256, 128),
200
- ).to(self.device)
201
-
202
- # Projection head disabled by default until trained with contrastive loss.
203
- # To enable: set REID_USE_PROJECTION=1
204
- use_projection = os.getenv('REID_USE_PROJECTION', '0') == '1'
205
- if use_projection:
206
- try:
207
- proj_path = resolve_weights('dinov3_reid_head.pth', local_dir=os.getenv('MODEL_PATH', '/models'))
208
- self.projection.load_state_dict(torch.load(proj_path, map_location=self.device))
209
- self.projection.eval()
210
- logger.info("Loaded contrastive Re-ID Projection Head (128-dim)")
211
- except FileNotFoundError:
212
- self.projection = None
213
- logger.warning("Re-ID projection head not found — falling back to base DINOv3 1536-dim embeddings")
214
- else:
215
- self.projection = None
216
- logger.info("Using base DINOv3 1536-dim embeddings (no projection head)")
217
-
218
- logger.info(f"DINOv3 Re-ID backbone loaded on {self.device}")
219
-
220
- def _crop_with_context(self, frame: np.ndarray, bbox: np.ndarray,
221
- context_ratio: float = 0.15) -> np.ndarray:
222
- """
223
- Crop equipment with context padding and square normalization.
224
- Adding 15% context around the bbox captures surrounding details
225
- (shadow pattern, ground type) that help stabilize embeddings.
226
- """
227
- x1, y1, x2, y2 = bbox[:4].astype(int)
228
- w, h = x2 - x1, y2 - y1
229
-
230
- # Add context padding
231
- pad_x = int(w * context_ratio)
232
- pad_y = int(h * context_ratio)
233
- x1 = max(0, x1 - pad_x)
234
- y1 = max(0, y1 - pad_y)
235
- x2 = min(frame.shape[1], x2 + pad_x)
236
- y2 = min(frame.shape[0], y2 + pad_y)
237
-
238
- crop = frame[y1:y2, x1:x2]
239
- if crop.size == 0 or crop.shape[0] < 20 or crop.shape[1] < 20:
240
- return None
241
-
242
- # Pad to square to avoid aspect-ratio distortion
243
- h, w = crop.shape[:2]
244
- if h != w:
245
- size = max(h, w)
246
- padded = np.zeros((size, size, 3), dtype=crop.dtype)
247
- y_off = (size - h) // 2
248
- x_off = (size - w) // 2
249
- padded[y_off:y_off + h, x_off:x_off + w] = crop
250
- crop = padded
251
-
252
- return crop
253
-
254
- @torch.inference_mode()
255
- def extract_embedding(self, frame: np.ndarray, bbox: np.ndarray) -> np.ndarray:
256
- """
257
- Extract a 1536-d embedding from an equipment crop.
258
- Uses CLS + mean(patch_tokens) with 15% context padding.
259
-
260
- Args:
261
- frame: Full BGR frame (H, W, 3)
262
- bbox: [x1, y1, x2, y2]
263
-
264
- Returns:
265
- Normalized embedding vector, or None
266
- """
267
- crop = self._crop_with_context(frame, bbox)
268
- if crop is None:
269
- return None
270
-
271
- crop_rgb = cv2.cvtColor(crop, cv2.COLOR_BGR2RGB)
272
- pil_img = Image.fromarray(crop_rgb)
273
- inputs = self.processor(images=pil_img, return_tensors="pt").to(self.device)
274
- outputs = self.model(**inputs)
275
-
276
- # CLS + mean(patch_tokens) — DINOv3 ViT-B/16: 1 CLS + 4 register + 196 patch
277
- hidden = outputs.last_hidden_state[0]
278
- cls_token = hidden[0]
279
- patch_tokens = hidden[5:]
280
- mean_patches = patch_tokens.mean(dim=0)
281
- embedding = torch.cat([cls_token, mean_patches])
282
-
283
- if self.projection is not None:
284
- import torch.nn.functional as F
285
- features = embedding.unsqueeze(0).to(self.device)
286
- projected = self.projection(features)
287
- projected = F.normalize(projected, p=2, dim=1)
288
- embedding = projected[0].cpu().numpy()
289
- else:
290
- embedding = embedding.cpu().numpy()
291
- norm = np.linalg.norm(embedding)
292
- if norm > 0:
293
- embedding = embedding / norm
294
-
295
- return embedding
296
-
297
- def _extract_color_histogram(self, frame: np.ndarray, bbox: np.ndarray) -> np.ndarray:
298
- """
299
- Extract an HSV color histogram from a crop. This is a hard physical
300
- discriminator: a blue truck and a white truck will NEVER match,
301
- regardless of what the neural network thinks.
302
- """
303
- x1, y1, x2, y2 = bbox[:4].astype(int)
304
- x1, y1 = max(0, x1), max(0, y1)
305
- x2 = min(frame.shape[1], x2)
306
- y2 = min(frame.shape[0], y2)
307
- crop = frame[y1:y2, x1:x2]
308
- if crop.size == 0:
309
- return None
310
- hsv = cv2.cvtColor(crop, cv2.COLOR_BGR2HSV)
311
- # 2D histogram on Hue (color) and Saturation (vividness)
312
- hist = cv2.calcHist([hsv], [0, 1], None, [30, 32], [0, 180, 0, 256])
313
- cv2.normalize(hist, hist)
314
- return hist.flatten()
315
-
316
- def update_active(self, track_id: int, embedding: np.ndarray,
317
- equipment_id: str, class_id: int, bbox: np.ndarray,
318
- frame_id: int):
319
- """Store embedding for an active track. Keeps up to 5 diverse views."""
320
- max_embeddings = 5
321
- if track_id not in self._active_embeddings:
322
- self._active_embeddings[track_id] = {
323
- 'embeddings': [embedding], # list of diverse views
324
- 'equipment_id': equipment_id,
325
- 'class_id': class_id,
326
- 'bbox': bbox.copy(),
327
- 'first_frame': frame_id,
328
- 'last_frame': frame_id,
329
- 'color_hist': None # populated by process_frame
330
- }
331
- else:
332
- entry = self._active_embeddings[track_id]
333
- entry['equipment_id'] = equipment_id
334
- entry['bbox'] = bbox.copy()
335
- entry['last_frame'] = frame_id
336
- # Only add if sufficiently different from existing embeddings
337
- existing = entry['embeddings']
338
- max_sim = max(float(np.dot(embedding, e)) for e in existing)
339
- if max_sim < 0.85 and len(existing) < max_embeddings:
340
- existing.append(embedding) # New diverse view
341
- elif len(existing) > 0:
342
- existing[-1] = embedding # Refresh the most recent one
343
-
344
- def move_to_gallery(self, track_id: int, frame_id: int):
345
- """
346
- Move a lost track to the gallery for future re-identification.
347
- Called when BoT-SORT drops a track.
348
- Only moves tracks that have an equipment_id (confirmed tracks).
349
- Pending tracks (no equipment_id yet) are silently dropped.
350
- """
351
- if track_id in self._active_embeddings:
352
- entry = self._active_embeddings.pop(track_id)
353
- # Only gallery tracks that have a confirmed equipment ID
354
- if entry.get('equipment_id') is None:
355
- return # pending track — never got an ID, just drop it
356
- entry['lost_frame'] = frame_id
357
- self._gallery[track_id] = entry
358
- logger.info(f"Track {track_id} ({entry['equipment_id']}) moved to Re-ID gallery (gallery size: {len(self._gallery)})")
359
-
360
- def _compute_spatial_similarity(self, query_bbox, gallery_bbox, frame_width, frame_height):
361
- """
362
- Compute spatial similarity between query and gallery bounding boxes.
363
- Uses both center distance and IoU-like size similarity.
364
- Returns 0.0 (far apart) to 1.0 (same position).
365
- """
366
- if query_bbox is None or gallery_bbox is None:
367
- return 0.0
368
-
369
- # Center distance (normalized by frame diagonal)
370
- lost_cx = (gallery_bbox[0] + gallery_bbox[2]) / 2
371
- lost_cy = (gallery_bbox[1] + gallery_bbox[3]) / 2
372
- new_cx = (query_bbox[0] + query_bbox[2]) / 2
373
- new_cy = (query_bbox[1] + query_bbox[3]) / 2
374
-
375
- diag = np.sqrt(frame_width**2 + frame_height**2)
376
- dist = np.sqrt((new_cx - lost_cx)**2 + (new_cy - lost_cy)**2)
377
- spatial_sim = max(0.0, 1.0 - dist / (diag * 0.5)) # 0 at half-diagonal away
378
-
379
- return spatial_sim
380
-
381
- def _get_alpha(self, frames_lost, fps=9.0):
382
- """
383
- Time-decaying alpha for combining visual + spatial similarity.
384
- For aerial drone footage, equipment barely moves during short track losses,
385
- so spatial proximity is the dominant signal. Visual appearance changes
386
- drastically with drone angle/lighting (same vehicle can score 0.40-0.80).
387
- """
388
- seconds_lost = frames_lost / fps
389
- if seconds_lost < 3.0:
390
- return 0.3 # 30% visual, 70% spatial — equipment hasn't moved
391
- elif seconds_lost < 10.0:
392
- return 0.5 # Equal weight — vehicle may have moved somewhat
393
- else:
394
- return 0.8 # Mostly visual — vehicle could be anywhere
395
-
396
- def query(self, embedding: np.ndarray, class_id: int,
397
- frame_id: int, query_bbox: np.ndarray = None,
398
- frame_width: int = 1920, frame_height: int = 1080,
399
- query_color_hist: np.ndarray = None) -> tuple:
400
- """
401
- Search the gallery for a matching lost track.
402
- Combines visual embedding similarity with spatial proximity,
403
- weighted by how long the track has been lost (time-decaying alpha).
404
-
405
- final_score = α * visual_sim + (1-α) * spatial_sim
406
- α increases with time lost (position becomes less reliable).
407
-
408
- Args:
409
- embedding: Query embedding from new detection
410
- class_id: Class of the new detection
411
- frame_id: Current frame number
412
- query_bbox: [x1,y1,x2,y2] of the new detection
413
- frame_width: Width of the video frame in pixels
414
- frame_height: Height of the video frame in pixels
415
- query_color_hist: HSV color histogram (unused, kept for API compat)
416
-
417
- Returns:
418
- (matched_equipment_id, final_score) or (None, 0.0)
419
- """
420
- if embedding is None or len(self._gallery) == 0:
421
- return None, 0.0
422
-
423
- best_match_id = None
424
- best_score = 0.0
425
- second_best_score = 0.0
426
- best_equip_id = None
427
- best_visual = 0.0
428
- best_spatial = 0.0
429
-
430
- expired = []
431
- for gal_track_id, entry in self._gallery.items():
432
- # Check TTL
433
- frames_lost = frame_id - entry.get('lost_frame', 0)
434
- if frames_lost > self.gallery_ttl_frames:
435
- expired.append(gal_track_id)
436
- continue
437
-
438
- # Only match same class
439
- if entry['class_id'] != class_id:
440
- continue
441
-
442
- # ── Visual Similarity (best of stored embeddings) ──
443
- visual_sim = max(
444
- float(np.dot(embedding, e)) for e in entry['embeddings']
445
- )
446
-
447
- # ── Spatial Similarity ──
448
- spatial_sim = self._compute_spatial_similarity(
449
- query_bbox, entry.get('bbox'), frame_width, frame_height
450
- )
451
-
452
- # ── Combined Score (time-decaying alpha) ──
453
- # BoT-SORT retains lost tracks for max_time_lost=1200 frames (20s at 60fps).
454
- # Gallery entries are for tracks BoT-SORT has already dropped.
455
- alpha = self._get_alpha(frames_lost)
456
- final_score = alpha * visual_sim + (1 - alpha) * spatial_sim
457
-
458
- # Hard floor: visual similarity must be at least 0.50
459
- # Prevents position-dominant matches between different same-class vehicles
460
- if visual_sim < 0.50:
461
- continue
462
-
463
- if final_score > best_score:
464
- # Track second-best for margin-based disambiguation
465
- second_best_score = best_score
466
- best_score = final_score
467
- best_match_id = gal_track_id
468
- best_equip_id = entry['equipment_id']
469
- best_visual = visual_sim
470
- best_spatial = spatial_sim
471
- elif final_score > second_best_score:
472
- second_best_score = final_score
473
-
474
- # Clean expired entries
475
- for tid in expired:
476
- del self._gallery[tid]
477
-
478
- # ── Threshold decision ──
479
- # BoT-SORT handles identity for up to 20s (max_time_lost=1200 at 60fps).
480
- # Re-ID only fires for tracks lost longer than that — be strict to avoid
481
- # false positives (different same-class vehicles getting the same ID).
482
- if best_match_id is not None:
483
- if best_score < self.similarity_threshold:
484
- best_match_id = None # reject — below threshold
485
-
486
- # Remove matched entry from gallery
487
- if best_match_id is not None:
488
- del self._gallery[best_match_id]
489
- logger.info(
490
- f"Re-ID match: {best_equip_id} "
491
- f"(combined={best_score:.3f}, visual={best_visual:.3f}, spatial={best_spatial:.3f})"
492
- )
493
- else:
494
- # Log why we didn't match
495
- if len(self._gallery) > 0:
496
- all_sims = []
497
- for gal_entry in self._gallery.values():
498
- if gal_entry['class_id'] == class_id:
499
- v_sim = max(float(np.dot(embedding, emb)) for emb in gal_entry['embeddings'])
500
- s_sim = self._compute_spatial_similarity(
501
- query_bbox, gal_entry.get('bbox'), frame_width, frame_height
502
- )
503
- fl = frame_id - gal_entry.get('lost_frame', 0)
504
- a = self._get_alpha(fl)
505
- combined = a * v_sim + (1 - a) * s_sim
506
- all_sims.append((gal_entry['equipment_id'], combined, v_sim, s_sim))
507
- if all_sims:
508
- best_below = max(all_sims, key=lambda x: x[1])
509
- logger.info(
510
- f"Re-ID no match: best={best_below[0]} "
511
- f"combined={best_below[1]:.3f} (visual={best_below[2]:.3f}, spatial={best_below[3]:.3f}), "
512
- f"threshold={self.similarity_threshold:.2f}"
513
- )
514
-
515
- return best_equip_id, best_score
516
-
517
- def absorb_near_misses(self, track_id: int, embedding: np.ndarray,
518
- class_id: int, frame_id: int,
519
- query_bbox: np.ndarray = None,
520
- frame_width: int = 1920, frame_height: int = 1080,
521
- id_generator=None):
522
- """
523
- When a new ID is minted (Re-ID missed), check if there are gallery entries
524
- of the same class that scored close to the threshold. If a high-confidence
525
- match is found (>0.85), retroactively reassign the new track to the old
526
- equipment ID — fixing phantom IDs (WL-003 → WL-002).
527
- For lower scores (>0.70), just absorb embeddings to help future matching.
528
- """
529
- near_miss_floor = self.similarity_threshold - 0.05
530
- candidates = []
531
-
532
- for gal_track_id, entry in list(self._gallery.items()):
533
- if entry['class_id'] != class_id:
534
- continue
535
-
536
- visual_sim = max(float(np.dot(embedding, e)) for e in entry['embeddings'])
537
- if visual_sim < 0.35:
538
- continue
539
-
540
- spatial_sim = self._compute_spatial_similarity(
541
- query_bbox, entry.get('bbox'), frame_width, frame_height
542
- )
543
- frames_lost = frame_id - entry.get('lost_frame', 0)
544
- alpha = self._get_alpha(frames_lost)
545
- combined = alpha * visual_sim + (1 - alpha) * spatial_sim
546
-
547
- if combined >= near_miss_floor:
548
- candidates.append((gal_track_id, entry, combined))
549
-
550
- if not candidates:
551
- return None
552
-
553
- # Sort by score — best match first
554
- candidates.sort(key=lambda x: x[2], reverse=True)
555
- best_gal_id, best_entry, best_score = candidates[0]
556
-
557
- # High-confidence near-miss: retroactively reassign ID
558
- # This fixes phantom IDs (e.g., WL-003 → WL-002 when score=0.975)
559
- reassigned_id = None
560
- if best_score >= 0.85 and id_generator is not None:
561
- old_equip_id = best_entry['equipment_id']
562
- current_equip_id = id_generator.get_existing(track_id)
563
- if current_equip_id and current_equip_id != old_equip_id:
564
- id_generator.reassign(track_id, old_equip_id)
565
- # Recycle the phantom ID number so next new vehicle reuses it
566
- id_generator.recycle(current_equip_id)
567
- # Update active embeddings with the corrected equipment ID
568
- if track_id in self._active_embeddings:
569
- self._active_embeddings[track_id]['equipment_id'] = old_equip_id
570
- reassigned_id = old_equip_id
571
- logger.info(
572
- f"Re-ID retroactive fix: {current_equip_id} → {old_equip_id} "
573
- f"(score={best_score:.3f}) — phantom ID corrected, "
574
- f"{current_equip_id} recycled"
575
- )
576
-
577
- # Absorb embeddings from all near-misses
578
- for gal_track_id, entry, combined in candidates:
579
- if track_id in self._active_embeddings:
580
- active = self._active_embeddings[track_id]
581
- for old_emb in entry['embeddings']:
582
- max_sim = max(float(np.dot(old_emb, e)) for e in active['embeddings'])
583
- if max_sim < 0.90 and len(active['embeddings']) < 8:
584
- active['embeddings'].append(old_emb)
585
- del self._gallery[gal_track_id]
586
-
587
- return reassigned_id
588
-
589
- def cleanup(self, frame_id: int, id_generator=None):
590
- """Remove expired gallery entries and recycle their equipment IDs.
591
- Self-healing: phantom IDs from false positives are recycled after TTL,
592
- making their numbers available for real equipment."""
593
- expired = [
594
- tid for tid, entry in self._gallery.items()
595
- if frame_id - entry['lost_frame'] > self.gallery_ttl_frames
596
- ]
597
- for tid in expired:
598
- entry = self._gallery[tid]
599
- equip_id = entry.get('equipment_id')
600
- if equip_id and id_generator:
601
- id_generator.recycle(equip_id)
602
- logger.info(f"Gallery TTL expired: {equip_id} recycled after {self.gallery_ttl_frames} frames")
603
- del self._gallery[tid]
604
-
605
-
606
- # =============================================================================
607
- # TAI Heuristic (Track-Aware Initialization from TrackTrack, CVPR 2025)
608
- # =============================================================================
609
- def apply_tai_heuristic(detections: np.ndarray,
610
- tracked_boxes: np.ndarray,
611
- ioa_threshold: float = 0.4) -> np.ndarray:
612
- """
613
- TAI port from TrackTrack (CVPR 2025).
614
- Filters detections that are spatially contained inside existing tracks,
615
- preventing fragmentary IDs (e.g., excavator bucket detected as separate equipment).
616
-
617
- Args:
618
- detections: (N, 6) array of [x1, y1, x2, y2, conf, class_id]
619
- tracked_boxes: (M, 4) array of [x1, y1, x2, y2] from active tracks
620
- ioa_threshold: Intersection-over-Area threshold (0.4 = 40% contained)
621
-
622
- Returns:
623
- Filtered detections (K, 6) where K <= N
624
- """
625
- if len(tracked_boxes) == 0 or len(detections) == 0:
626
- return detections
627
-
628
- filtered = []
629
- for det in detections:
630
- det_box = det[:4]
631
-
632
- # Calculate IoA: intersection / area_of_detection
633
- # (How much of the detection is inside an existing track?)
634
- det_area = max((det_box[2] - det_box[0]) * (det_box[3] - det_box[1]), 1e-6)
635
-
636
- is_fragment = False
637
- for track_box in tracked_boxes:
638
- track_area = max(
639
- (track_box[2] - track_box[0]) * (track_box[3] - track_box[1]), 1e-6
640
- )
641
-
642
- # Only consider fragment filtering if the detection is
643
- # significantly SMALLER than the track (< 50% of track area).
644
- # Full-size re-detections of the same equipment should pass through.
645
- if det_area > 0.5 * track_area:
646
- continue
647
-
648
- # Intersection
649
- x1 = max(det_box[0], track_box[0])
650
- y1 = max(det_box[1], track_box[1])
651
- x2 = min(det_box[2], track_box[2])
652
- y2 = min(det_box[3], track_box[3])
653
-
654
- inter_area = max(0, x2 - x1) * max(0, y2 - y1)
655
- ioa = inter_area / det_area
656
-
657
- if ioa > ioa_threshold:
658
- is_fragment = True
659
- break
660
-
661
- if not is_fragment:
662
- filtered.append(det)
663
-
664
- if len(filtered) == 0:
665
- return np.empty((0, 6))
666
- return np.array(filtered)
667
-
668
-
669
- # =============================================================================
670
- # Grid-Based Articulated Motion Detection (Farneback Optical Flow)
671
- # =============================================================================
672
- class ArticulatedMotionDetector:
673
- """
674
- Detects motion within equipment bounding boxes using
675
- Grid Decomposition (3x3) + Farneback Dense Optical Flow.
676
-
677
- Equipment-agnostic: works for excavators (arm-only motion),
678
- wheel loaders (bucket pivot), dump trucks (driving), etc.
679
- No assumptions about which grid region corresponds to which part.
680
- """
681
-
682
- def __init__(self, grid_size: int = 3, motion_threshold: float = 0.15):
683
- self.grid_size = grid_size
684
- self.motion_threshold = motion_threshold
685
- self._prev_crops = {} # track_id -> previous grayscale crop
686
- self._centroid_history = {} # track_id -> deque of (cx, cy)
687
- self._centroid_window = 30 # 0.5s at 60fps
688
- self._active_history = {} # track_id -> deque of recent is_active bools
689
- self._active_smoothing = 60 # 1 second at 60fps — bridges intra-cycle pauses
690
-
691
- def detect(self, frame_gray: np.ndarray, track_id: int,
692
- bbox: np.ndarray) -> dict:
693
- """
694
- Analyze motion within a bounding box.
695
-
696
- Returns:
697
- {
698
- 'is_active': bool,
699
- 'motion_source': str ('partial', 'whole_body', 'none'),
700
- 'cell_motions': list[list[float]], # 3x3 magnitude grid
701
- 'cell_directions': list[list[list[float]]], # 3x3x2 flow direction grid
702
- 'max_flow': float,
703
- 'active_cell_ratio': float # fraction of cells with motion
704
- }
705
- """
706
- x1, y1, x2, y2 = bbox[:4].astype(int)
707
- x1, y1 = max(0, x1), max(0, y1)
708
- x2 = min(frame_gray.shape[1], x2)
709
- y2 = min(frame_gray.shape[0], y2)
710
-
711
- # --- Centroid displacement over rolling window ---
712
- cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
713
- centroid_moving = False
714
- if track_id not in self._centroid_history:
715
- self._centroid_history[track_id] = deque(maxlen=self._centroid_window + 1)
716
- self._centroid_history[track_id].append((cx, cy))
717
-
718
- history = self._centroid_history[track_id]
719
- if len(history) >= 2:
720
- old_cx, old_cy = history[0]
721
- displacement = np.sqrt((cx - old_cx) ** 2 + (cy - old_cy) ** 2)
722
- if displacement > 15.0:
723
- centroid_moving = True
724
-
725
- crop = frame_gray[y1:y2, x1:x2]
726
-
727
- if crop.size == 0 or crop.shape[0] < 10 or crop.shape[1] < 10:
728
- return self._empty_result(centroid_moving)
729
-
730
- # Resize crop to standard size — this inherently normalizes for scale.
731
- # Large boxes (close equipment) and small boxes (distant) produce
732
- # comparable flow magnitudes after resize. No additional normalization needed.
733
- crop_resized = self._resize_crop(crop, 300)
734
-
735
- if track_id not in self._prev_crops:
736
- self._prev_crops[track_id] = crop_resized
737
- return self._empty_result(centroid_moving)
738
-
739
- prev_crop = self._prev_crops[track_id]
740
- self._prev_crops[track_id] = crop_resized
741
-
742
- if prev_crop.shape != crop_resized.shape:
743
- return self._empty_result(centroid_moving)
744
-
745
- # Farneback dense optical flow
746
- flow = cv2.calcOpticalFlowFarneback(
747
- prev_crop, crop_resized,
748
- None,
749
- pyr_scale=0.5, levels=3, winsize=15,
750
- iterations=3, poly_n=5, poly_sigma=1.2,
751
- flags=0
752
- )
753
-
754
- # ── Global motion detection (ACTIVE/INACTIVE) ──
755
- # Dual-gate approach: detect motion via EITHER of two independent signals.
756
- # This prevents the failure mode where one metric collapses while
757
- # the equipment is clearly active.
758
- all_magnitudes = np.sqrt(flow[..., 0] ** 2 + flow[..., 1] ** 2)
759
- top_flow = float(np.percentile(all_magnitudes, 95))
760
- median_flow = float(np.median(all_magnitudes))
761
- localized_motion = top_flow - median_flow
762
-
763
- # Gate 1: Localized motion (top5% - median > threshold)
764
- # Catches articulated motion (arm/bucket) when background is stable.
765
- is_localized = localized_motion > self.motion_threshold
766
-
767
- # Gate 2: Absolute flow magnitude (top5% > 0.35)
768
- # Catches ALL significant motion regardless of median elevation.
769
- # Parked equipment: top5% ≈ 0.14-0.29 (max observed 0.29).
770
- # Bucket-only tip-in-place: top5% ≈ 0.35-0.60.
771
- # Active equipment: top5% ≈ 0.5-1.2.
772
- # 0.35 gives 0.06 margin above highest parked value while catching
773
- # the tip-in-place phase that was draining the counter.
774
- is_strong_flow = top_flow > 0.35
775
-
776
- is_active_flow = is_localized or is_strong_flow
777
-
778
- # ── Grid decomposition (for activity classification) ──
779
- h, w = crop_resized.shape[:2]
780
- cell_h, cell_w = h // self.grid_size, w // self.grid_size
781
-
782
- cell_motions = []
783
- cell_directions = []
784
-
785
- for row in range(self.grid_size):
786
- motion_row = []
787
- direction_row = []
788
- for col in range(self.grid_size):
789
- cy1 = row * cell_h
790
- cy2 = (row + 1) * cell_h if row < self.grid_size - 1 else h
791
- cx1 = col * cell_w
792
- cx2 = (col + 1) * cell_w if col < self.grid_size - 1 else w
793
-
794
- cell_flow = flow[cy1:cy2, cx1:cx2]
795
- magnitudes = np.sqrt(
796
- cell_flow[..., 0] ** 2 + cell_flow[..., 1] ** 2
797
- )
798
-
799
- p95_mag = float(np.percentile(magnitudes, 95))
800
- motion_row.append(p95_mag)
801
-
802
- avg_dx = float(np.mean(cell_flow[..., 0]))
803
- avg_dy = float(np.mean(cell_flow[..., 1]))
804
- direction_row.append([avg_dx, avg_dy])
805
-
806
- cell_motions.append(motion_row)
807
- cell_directions.append(direction_row)
808
-
809
- flat = np.array(cell_motions)
810
- active_cells = flat > self.motion_threshold
811
- num_active = int(np.sum(active_cells))
812
- total_cells = self.grid_size * self.grid_size
813
- active_ratio = num_active / total_cells
814
-
815
- raw_active = is_active_flow or centroid_moving
816
-
817
- # ── Asymmetric hysteresis ──
818
- # Construction equipment works in cycles (scoop-lift-dump-return).
819
- # Brief pauses between phases should NOT flip state to INACTIVE.
820
- # INACTIVE → ACTIVE: need 3/15 recent frames active (20%) — respond quickly
821
- # ACTIVE → INACTIVE: need 12/15 recent frames inactive (80%) — hold through pauses
822
- if track_id not in self._active_history:
823
- self._active_history[track_id] = deque(maxlen=self._active_smoothing)
824
- self._active_history[track_id].append(raw_active)
825
- active_votes = sum(self._active_history[track_id])
826
- total_votes = len(self._active_history[track_id])
827
-
828
- # Asymmetric thresholds based on current state
829
- if not hasattr(self, '_prev_smoothed'):
830
- self._prev_smoothed = {}
831
- prev_active = self._prev_smoothed.get(track_id, False)
832
- if prev_active:
833
- # Currently ACTIVE — slow decay (stay active if ≥8/60 recent frames active)
834
- is_active = active_votes >= 8
835
- else:
836
- # Currently INACTIVE — fast attack (activate if ≥3/15 recent frames active)
837
- recent_votes = sum(list(self._active_history[track_id])[-15:])
838
- is_active = recent_votes >= 3
839
- self._prev_smoothed[track_id] = is_active
840
-
841
- # Motion source classification
842
- if centroid_moving or active_ratio > 0.5:
843
- motion_source = 'whole_body'
844
- elif is_active:
845
- motion_source = 'partial'
846
- else:
847
- motion_source = 'none'
848
-
849
- # Debug logging
850
- if not hasattr(self, '_debug_counter'):
851
- self._debug_counter = 0
852
- self._debug_counter += 1
853
- if self._debug_counter % 30 == 0:
854
- gate = "LOC" if is_localized else ("ABS" if is_strong_flow else ("CEN" if centroid_moving else "---"))
855
- logger.info(
856
- f"Motion[T{track_id}]: top5%={top_flow:.3f}, localized={localized_motion:.3f}, "
857
- f"gate={gate}, raw={raw_active}, smoothed={is_active} ({active_votes}/{total_votes}), "
858
- f"crop={crop_resized.shape}"
859
- )
860
-
861
- return {
862
- 'is_active': is_active,
863
- 'motion_source': motion_source,
864
- 'cell_motions': cell_motions,
865
- 'cell_directions': cell_directions,
866
- 'max_flow': top_flow,
867
- 'localized_motion': localized_motion,
868
- 'active_cell_ratio': active_ratio
869
- }
870
-
871
- def remove_track(self, track_id: int):
872
- self._prev_crops.pop(track_id, None)
873
- self._centroid_history.pop(track_id, None)
874
- self._active_history.pop(track_id, None)
875
- if hasattr(self, '_prev_smoothed'):
876
- self._prev_smoothed.pop(track_id, None)
877
-
878
- def _resize_crop(self, crop: np.ndarray, target_size: int) -> np.ndarray:
879
- return cv2.resize(crop, (target_size, target_size), interpolation=cv2.INTER_LINEAR)
880
-
881
- def _empty_result(self, centroid_moving: bool = False) -> dict:
882
- empty_grid = [[0.0] * self.grid_size for _ in range(self.grid_size)]
883
- empty_dirs = [[[0.0, 0.0]] * self.grid_size for _ in range(self.grid_size)]
884
- return {
885
- 'is_active': centroid_moving,
886
- 'motion_source': 'whole_body' if centroid_moving else 'none',
887
- 'cell_motions': empty_grid,
888
- 'cell_directions': empty_dirs,
889
- 'max_flow': 0.0,
890
- 'localized_motion': 0.0,
891
- 'active_cell_ratio': 0.0
892
- }
893
-
894
-
895
- # =============================================================================
896
- # Rule-Based Activity Classifier
897
- # =============================================================================
898
- class RuleBasedActivityClassifier:
899
- """
900
- Per-track classifier: only determines ACTIVE/INACTIVE → WAITING.
901
-
902
- Activity sub-labels (DIGGING, LOADING, DUMPING) are assigned in Phase 10
903
- using spatial context — specifically, the equipment's proximity to dump
904
- trucks. The flow ratio cannot distinguish DIGGING from DUMPING because
905
- both produce identical signal topology (concentrated bucket motion).
906
- Spatial context resolves this: at the pile = DIGGING, at the truck = DUMPING.
907
- """
908
-
909
- def __init__(self):
910
- pass
911
-
912
- def classify_smoothed(self, track_id: int, motion_result: dict,
913
- equipment_class: str = '') -> str:
914
- """Return WAITING if inactive, ACTIVE placeholder otherwise.
915
- Phase 10 will assign the real activity sub-label."""
916
- if not motion_result['is_active']:
917
- return 'WAITING'
918
- # Placeholder — Phase 10 spatial context replaces this
919
- return 'LOADING'
920
-
921
- def remove_track(self, track_id: int):
922
- pass
923
 
 
 
 
 
 
924
 
925
- # =============================================================================
926
- # Video Activity Classifier (Option B — X3D-S)
927
- # =============================================================================
928
- class VideoActivityClassifier:
929
- """
930
- Classifies equipment activity from 16-frame crop buffers using a
931
- fine-tuned X3D-S model (Kinetics-400 pretrained → 4 activity classes).
932
 
933
- Maintains a sliding window of cropped frames per track. When the buffer
934
- reaches `clip_length` frames, runs inference and returns the predicted
935
- activity label. Falls back to rule-based Phase 10 if model is not loaded.
936
 
937
- Activities: DIGGING, LOADING, DUMPING, WAITING
938
- """
939
 
940
- ACTIVITY_CLASSES = ['DIGGING', 'LOADING', 'DUMPING', 'WAITING']
941
 
942
- def __init__(self, weights_path: str = None, clip_length: int = 16,
943
- clip_stride: int = 8, crop_size: int = 224,
944
- confidence_threshold: float = 0.5, device: str = 'cuda'):
945
- self.clip_length = clip_length
946
- self.clip_stride = clip_stride
947
- self.crop_size = crop_size
948
- self.confidence_threshold = confidence_threshold
949
- self.device = device if torch.cuda.is_available() else 'cpu'
950
- self.model = None
951
- self.weights_path = weights_path
952
 
953
- # Per-track crop buffers: track_id -> list of (H,W,3) uint8 crops
954
- self._buffers = {}
955
- # Per-track last prediction: track_id -> (activity, confidence)
956
- self._last_pred = {}
957
 
958
- # Normalization (Kinetics standard)
959
- self._mean = torch.tensor([0.45, 0.45, 0.45]).view(1, 3, 1, 1)
960
- self._std = torch.tensor([0.225, 0.225, 0.225]).view(1, 3, 1, 1)
961
 
962
- def load(self):
963
- """Load the fine-tuned X3D-S model."""
964
- if not self.weights_path or not os.path.exists(self.weights_path):
965
- logger.warning(f"Activity classifier weights not found: {self.weights_path}")
966
- logger.warning("Falling back to rule-based Phase 10 classification")
967
- return False
968
 
969
- try:
970
- from pytorchvideo.models.hub import x3d_s
971
- import torch.nn as nn
 
 
 
 
 
 
972
 
973
- checkpoint = torch.load(self.weights_path, map_location=self.device)
974
 
975
- self.model = x3d_s(pretrained=False)
976
- in_features = self.model.blocks[5].proj.in_features
977
- self.model.blocks[5].proj = nn.Linear(in_features, len(self.ACTIVITY_CLASSES))
978
- self.model.load_state_dict(checkpoint['model_state_dict'])
979
- self.model = self.model.to(self.device)
980
- self.model.eval()
981
 
982
- # Move normalization tensors to device
983
- self._mean = self._mean.to(self.device)
984
- self._std = self._std.to(self.device)
985
 
986
- val_acc = checkpoint.get('best_val_acc', 'unknown')
987
- logger.info(f"Activity classifier loaded: X3D-S, val_acc={val_acc}%")
988
- return True
989
- except Exception as e:
990
- logger.error(f"Failed to load activity classifier: {e}")
991
- self.model = None
992
- return False
993
 
994
- @property
995
- def is_loaded(self):
996
- return self.model is not None
997
 
998
- def feed_crop(self, track_id: int, crop: np.ndarray):
999
- """
1000
- Add a cropped frame to the track's buffer.
1001
- Args:
1002
- track_id: Tracker-assigned ID
1003
- crop: BGR crop from cv2, any size (will be resized)
1004
- """
1005
- if crop.size == 0 or crop.shape[0] < 10 or crop.shape[1] < 10:
1006
- return
1007
 
1008
- # Resize to standard input size
1009
- resized = cv2.resize(crop, (self.crop_size, self.crop_size))
1010
- # BGR → RGB
1011
- resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
1012
 
1013
- if track_id not in self._buffers:
1014
- self._buffers[track_id] = []
1015
- self._buffers[track_id].append(resized)
1016
 
1017
- def classify(self, track_id: int) -> tuple:
1018
- """
1019
- Classify the track's activity if enough frames are buffered.
 
 
1020
 
1021
- Returns:
1022
- (activity_label, confidence) or (None, 0.0) if not enough frames.
1023
- activity_label is one of: DIGGING, LOADING, DUMPING, WAITING
1024
- """
1025
- if not self.is_loaded:
1026
- return None, 0.0
1027
 
1028
- buf = self._buffers.get(track_id, [])
1029
- if len(buf) < self.clip_length:
1030
- # Not enough frames yet — return last prediction if available
1031
- return self._last_pred.get(track_id, (None, 0.0))
1032
 
1033
- # Take the most recent clip_length frames
1034
- clip_frames = buf[-self.clip_length:]
 
 
1035
 
1036
- # Convert to tensor: (T, H, W, 3) uint8 → (1, C, T, H, W) float
1037
- frames_np = np.stack(clip_frames, axis=0) # (T, H, W, 3)
1038
- clip_tensor = torch.from_numpy(frames_np).float() / 255.0 # (T, H, W, C)
1039
- clip_tensor = clip_tensor.permute(3, 0, 1, 2) # (C, T, H, W)
1040
 
1041
- # Normalize
1042
- clip_tensor = (clip_tensor - self._mean.cpu()) / self._std.cpu()
1043
- clip_tensor = clip_tensor.unsqueeze(0).to(self.device) # (1, C, T, H, W)
1044
 
1045
- # Inference
1046
- with torch.no_grad():
1047
- logits = self.model(clip_tensor)
1048
- probs = torch.softmax(logits, dim=1)
1049
- pred_idx = probs.argmax(1).item()
1050
- confidence = probs[0, pred_idx].item()
1051
 
1052
- activity = self.ACTIVITY_CLASSES[pred_idx]
 
1053
 
1054
- # Only trust prediction if confidence is above threshold
1055
- if confidence < self.confidence_threshold:
1056
- # Low confidence — keep previous prediction or default
1057
- prev = self._last_pred.get(track_id, (None, 0.0))
1058
- if prev[0] is not None:
1059
- activity, confidence = prev
1060
 
1061
- self._last_pred[track_id] = (activity, confidence)
 
 
 
1062
 
1063
- # Slide the buffer forward
1064
- self._buffers[track_id] = buf[self.clip_stride:]
1065
 
1066
- return activity, confidence
1067
 
1068
- def remove_track(self, track_id: int):
1069
- """Clean up state for a lost track."""
1070
- self._buffers.pop(track_id, None)
1071
- self._last_pred.pop(track_id, None)
1072
 
 
 
 
 
 
 
 
 
1073
 
1074
- # =============================================================================
1075
- # Equipment ID Generator
1076
- # =============================================================================
1077
- class EquipmentIDGenerator:
1078
- """Generates class-prefixed equipment IDs: EX-001, DT-001, BD-001 (per-class counters)."""
1079
-
1080
- # Minimum confidence to hard-lock class at creation. Below this,
1081
- # class can be corrected by a majority vote over the first N frames.
1082
- CLASS_LOCK_CONF_THRESHOLD = 0.55
1083
-
1084
- def __init__(self):
1085
- # Audit #7: per-class counters instead of global — avoids inflated IDs
1086
- self._counters = {} # prefix -> int
1087
- self._track_to_equip = {} # tracker_id -> equipment_id
1088
- self._equip_to_class = {} # equipment_id -> class_id (locked at creation)
1089
- self._class_locked = {} # equipment_id -> bool (True = hard-locked)
1090
- self._class_votes = {} # equipment_id -> list of (class_id, conf) for soft-lock
1091
- self._recycled = {} # prefix -> sorted list of recycled numbers
1092
-
1093
- def get_existing(self, track_id: int) -> str | None:
1094
- """Get existing equipment ID for this track, or None if new."""
1095
- return self._track_to_equip.get(track_id)
1096
-
1097
- def get_or_create(self, track_id: int, class_id: int, conf: float = 1.0) -> str:
1098
- """Get existing equipment ID or create a new one for this track."""
1099
- if track_id in self._track_to_equip:
1100
- return self._track_to_equip[track_id]
1101
-
1102
- prefix = CLASS_PREFIXES.get(int(class_id), 'UK')
1103
- self._counters.setdefault(prefix, 0)
1104
-
1105
- # Use recycled number if available, otherwise increment counter
1106
- if prefix in self._recycled and self._recycled[prefix]:
1107
- num = self._recycled[prefix].pop(0)
1108
- else:
1109
- self._counters[prefix] += 1
1110
- num = self._counters[prefix]
1111
-
1112
- equip_id = f"{prefix}-{num:03d}"
1113
- self._track_to_equip[track_id] = equip_id
1114
- self._equip_to_class[equip_id] = class_id
1115
-
1116
- # Hard-lock class only if initial detection confidence is high enough
1117
- if conf >= self.CLASS_LOCK_CONF_THRESHOLD:
1118
- self._class_locked[equip_id] = True
1119
- else:
1120
- self._class_locked[equip_id] = False
1121
- self._class_votes[equip_id] = [(class_id, conf)]
1122
- logger.info(f"{equip_id} class soft-locked (conf={conf:.2f} < {self.CLASS_LOCK_CONF_THRESHOLD}), may be corrected")
1123
-
1124
- return equip_id
1125
-
1126
- def reassign(self, new_track_id: int, equip_id: str):
1127
- """Reassign an equipment ID to a new track (after Re-ID match)."""
1128
- self._track_to_equip[new_track_id] = equip_id
1129
-
1130
- def recycle(self, equip_id: str):
1131
- """Recycle a phantom equipment ID so its number can be reused.
1132
- Called when absorb_near_misses reassigns a phantom to an older ID."""
1133
- parts = equip_id.split('-')
1134
- if len(parts) == 2:
1135
- prefix = parts[0]
1136
- num = int(parts[1])
1137
- self._recycled.setdefault(prefix, [])
1138
- if num not in self._recycled[prefix]:
1139
- self._recycled[prefix].append(num)
1140
- self._recycled[prefix].sort()
1141
- # Clean up the phantom from equip_to_class
1142
- self._equip_to_class.pop(equip_id, None)
1143
- logger.info(f"Recycled phantom ID {equip_id} — number available for reuse")
1144
-
1145
- def get_locked_class(self, equip_id: str) -> int | None:
1146
- """Get the class_id that was locked when this equipment ID was created."""
1147
- return self._equip_to_class.get(equip_id)
1148
-
1149
- def is_class_locked(self, equip_id: str) -> bool:
1150
- """True if this equipment's class is hard-locked (high initial confidence)."""
1151
- return self._class_locked.get(equip_id, True)
1152
-
1153
- def vote_class(self, track_id: int, equip_id: str, class_id: int, conf: float):
1154
- """For soft-locked equipment, accumulate class votes over first 10 detections.
1155
- After 10 votes, pick the weighted majority class and hard-lock it.
1156
- If the majority differs from the initial class, reassign the equipment ID."""
1157
- if self._class_locked.get(equip_id, True):
1158
- return # already hard-locked
1159
-
1160
- votes = self._class_votes.get(equip_id, [])
1161
- votes.append((class_id, conf))
1162
- self._class_votes[equip_id] = votes
1163
-
1164
- # After 10 observations, hard-lock to weighted majority class
1165
- if len(votes) >= 10:
1166
- # Weighted vote: sum confidence per class
1167
- class_weights = {}
1168
- for cid, c in votes:
1169
- class_weights[cid] = class_weights.get(cid, 0.0) + c
1170
- winner = max(class_weights, key=class_weights.get)
1171
- current_class = self._equip_to_class.get(equip_id)
1172
-
1173
- if winner != current_class:
1174
- # Class correction needed — change the equipment ID prefix
1175
- old_prefix = CLASS_PREFIXES.get(int(current_class), 'UK')
1176
- new_prefix = CLASS_PREFIXES.get(int(winner), 'UK')
1177
-
1178
- # Recycle the old number
1179
- old_parts = equip_id.split('-')
1180
- old_num = int(old_parts[1])
1181
- self._recycled.setdefault(old_prefix, [])
1182
- if old_num not in self._recycled[old_prefix]:
1183
- self._recycled[old_prefix].append(old_num)
1184
- self._recycled[old_prefix].sort()
1185
-
1186
- # Allocate new number with correct prefix
1187
- self._counters.setdefault(new_prefix, 0)
1188
- if new_prefix in self._recycled and self._recycled[new_prefix]:
1189
- new_num = self._recycled[new_prefix].pop(0)
1190
- else:
1191
- self._counters[new_prefix] += 1
1192
- new_num = self._counters[new_prefix]
1193
-
1194
- new_equip_id = f"{new_prefix}-{new_num:03d}"
1195
- self._track_to_equip[track_id] = new_equip_id
1196
- self._equip_to_class[new_equip_id] = winner
1197
- self._class_locked[new_equip_id] = True
1198
- # Clean up old
1199
- self._equip_to_class.pop(equip_id, None)
1200
- self._class_locked.pop(equip_id, None)
1201
- self._class_votes.pop(equip_id, None)
1202
- logger.info(
1203
- f"Class correction: {equip_id} ({CLASS_NAMES.get(current_class)}) → "
1204
- f"{new_equip_id} ({CLASS_NAMES.get(winner)}) after {len(votes)} votes"
1205
- )
1206
- else:
1207
- # Same class confirmed — hard-lock it
1208
- self._class_locked[equip_id] = True
1209
- self._class_votes.pop(equip_id, None)
1210
- logger.info(f"{equip_id} class confirmed and hard-locked after {len(votes)} votes")
1211
-
1212
- def is_active(self, equip_id: str, exclude_track: int = None,
1213
- active_track_ids: set = None) -> bool:
1214
- """Check if an equipment ID is currently assigned to any active track.
1215
- active_track_ids: set of track IDs present in the current frame.
1216
- Only these are truly 'active' — gallery tracks are not."""
1217
- if active_track_ids is None:
1218
- return False # Can't determine without active set
1219
- for tid, eid in self._track_to_equip.items():
1220
- if eid == equip_id and tid != exclude_track and tid in active_track_ids:
1221
- return True
1222
- return False
1223
-
1224
- def remove(self, track_id: int):
1225
- self._track_to_equip.pop(track_id, None)
1226
-
1227
-
1228
- # =============================================================================
1229
- # Kafka Telemetry Producer
1230
- # =============================================================================
1231
- class TelemetryProducer:
1232
- """Produces equipment telemetry events to Kafka."""
1233
-
1234
- def __init__(self, bootstrap_servers: str, topic: str):
1235
- self.topic = topic
1236
- self._producer = None
1237
- self._bootstrap_servers = bootstrap_servers
1238
-
1239
- def connect(self):
1240
- from confluent_kafka import Producer
1241
- self._producer = Producer({
1242
- 'bootstrap.servers': self._bootstrap_servers,
1243
- 'enable.idempotence': True,
1244
- 'acks': 'all',
1245
- 'linger.ms': 5,
1246
- 'batch.num.messages': 100,
1247
- })
1248
- logger.info(f"Kafka producer connected: {self._bootstrap_servers}")
1249
-
1250
- def produce(self, equipment_id: str, payload: dict):
1251
- """Produce a single telemetry event, keyed by equipment_id."""
1252
- if self._producer is None:
1253
- return
1254
-
1255
- self._producer.produce(
1256
- topic=self.topic,
1257
- key=equipment_id.encode('utf-8'),
1258
- value=json.dumps(payload).encode('utf-8'),
1259
- callback=self._delivery_callback
1260
- )
1261
- self._producer.poll(0)
1262
-
1263
- def flush(self):
1264
- if self._producer:
1265
- self._producer.flush(timeout=5.0)
1266
-
1267
- @staticmethod
1268
- def _delivery_callback(err, msg):
1269
- if err:
1270
- logger.error(f"Kafka delivery failed: {err}")
1271
-
1272
-
1273
- # =============================================================================
1274
- # Main Pipeline Orchestrator
1275
- # =============================================================================
1276
- class InferencePipeline:
1277
- """
1278
- Main pipeline class that orchestrates all CV components.
1279
-
1280
- Flow: Frame → Detect → TAI Filter → Track → Re-ID → Motion → Activity → Kafka
1281
- """
1282
-
1283
- def __init__(self, config: dict):
1284
- self.config = config
1285
-
1286
- # Core ML components (initialized in .initialize())
1287
- self.detector = None
1288
- self.tracker = None
1289
- self.reid_gallery = None
1290
-
1291
- # Analysis components
1292
- self.motion_detector = ArticulatedMotionDetector(
1293
- grid_size=3,
1294
- motion_threshold=config.get('motion_threshold', 0.5) # Audit #3: match class default
1295
- )
1296
- # Activity classification is handled by Phase 10 (spatial context).
1297
- # RuleBasedActivityClassifier kept for remove_track cleanup only.
1298
- self.activity_classifier = RuleBasedActivityClassifier()
1299
-
1300
- # Option B: Video-based activity classifier (X3D-S)
1301
- # Falls back to rule-based Phase 10 if weights not available.
1302
- self.video_classifier = VideoActivityClassifier(
1303
- weights_path=config.get('activity_classifier_weights', 'weights/activity_classifier_x3d_s.pt'),
1304
- clip_length=16,
1305
- clip_stride=8,
1306
- crop_size=224,
1307
- confidence_threshold=0.5,
1308
- device='cuda',
1309
- )
1310
-
1311
- self.id_generator = EquipmentIDGenerator()
1312
- self.telemetry_producer = TelemetryProducer(
1313
- bootstrap_servers=config.get('kafka_servers', 'kafka:9092'),
1314
- topic=config.get('kafka_topic', 'equipment.telemetry.raw')
1315
- )
1316
-
1317
- # Frame counter for skip strategies
1318
- self._frame_count = 0
1319
- # Track IDs from previous frame (for detecting lost tracks)
1320
- self._prev_track_ids = set()
1321
- # Track first-seen frame for deferred ID creation.
1322
- # Tracks must persist for MIN_TRACK_FRAMES before getting an equipment ID.
1323
- # This prevents transient false detections from consuming ID numbers.
1324
- self._track_first_seen = {} # track_id -> frame_id
1325
- self.MIN_TRACK_FRAMES = 30 # 0.5s at 60fps
1326
- # Activity smoothing
1327
- self._activity_history = {} # equip_id -> deque(...)
1328
-
1329
- def initialize(self):
1330
- """Initialize all components."""
1331
- logger.info("Initializing inference pipeline...")
1332
-
1333
- import shutil
1334
- model_dir = os.getenv('MODEL_PATH', '/models')
1335
-
1336
- # Phase 3: RF-DETR Detector
1337
- weights_path = resolve_weights('rfdetr_construction.pth', local_dir=model_dir)
1338
-
1339
- # Prevent base model redownloads by copying the user mapped file
1340
- user_model = os.path.join(model_dir, "rf-detr-base.pth")
1341
- if os.path.exists(user_model) and not os.path.exists("rf-detr-base.pth"):
1342
- try:
1343
- shutil.copy2(user_model, "rf-detr-base.pth")
1344
- logger.info("Found user's manually downloaded rf-detr-base.pth. Bypassing download!")
1345
- except Exception as e:
1346
- pass
1347
- self.detector = RFDETRDetector(
1348
- weights_path=weights_path,
1349
- confidence_threshold=self.config.get('confidence_threshold', 0.35),
1350
- device=self.config.get('device', 'cuda')
1351
- )
1352
- self.detector.load()
1353
-
1354
- # Phase 4: BoT-SORT Tracker
1355
- osnet_path = resolve_weights('osnet_x0_25_msmt17.pt', local_dir=model_dir)
1356
-
1357
- # Copy to boxmot's expected location to avoid re-downloading
1358
- reid_dst = "/usr/local/lib/python3.10/dist-packages/models/osnet_x0_25_msmt17.pt"
1359
- if os.path.exists(osnet_path) and not os.path.exists(reid_dst):
1360
- os.makedirs(os.path.dirname(reid_dst), exist_ok=True)
1361
- shutil.copy2(osnet_path, reid_dst)
1362
- logger.info("Copied cached ReID weights — skipping download")
1363
-
1364
- from boxmot import create_tracker
1365
- botsort_config = Path("/configs/botsort.yaml")
1366
- self.tracker = create_tracker(
1367
- 'botsort',
1368
- tracker_config=botsort_config,
1369
- reid_weights=Path(reid_dst if os.path.exists(reid_dst) else osnet_path),
1370
- device='0',
1371
- half=True
1372
- )
1373
- # Log actual tracker params to verify config was parsed correctly
1374
- max_time_lost = getattr(self.tracker, 'max_time_lost', '?')
1375
- buffer_size = getattr(self.tracker, 'buffer_size', '?')
1376
- logger.info(
1377
- f"BoT-SORT initialized: max_time_lost={max_time_lost}, "
1378
- f"buffer_size={buffer_size}"
1379
- )
1380
-
1381
- # Phase 5: DINOv3 Re-ID Gallery
1382
- self.reid_gallery = DINOv3ReIDGallery(
1383
- similarity_threshold=self.config.get('reid_threshold', 0.75),
1384
- gallery_ttl_frames=self.config.get('reid_gallery_ttl', 1800),
1385
- device=self.config.get('device', 'cuda'),
1386
- spatial_gate_ratio=0.60,
1387
- )
1388
- self.reid_gallery.load()
1389
-
1390
- # Option B: Video activity classifier (X3D-S)
1391
- if self.video_classifier.load():
1392
- logger.info("Video activity classifier loaded — will override Phase 10 rules")
1393
- else:
1394
- logger.info("Video activity classifier not available — using rule-based Phase 10")
1395
-
1396
- # Kafka producer
1397
- self.telemetry_producer.connect()
1398
-
1399
- logger.info("Pipeline initialized — all components ready.")
1400
-
1401
- def process_frame(self, frame: np.ndarray, frame_id: int,
1402
- timestamp: float, source_id: str = "cam_01",
1403
- fps: float = 30.0):
1404
- """
1405
- Process a single frame through the full pipeline.
1406
-
1407
- Args:
1408
- frame: BGR image (H, W, 3)
1409
- frame_id: Frame counter
1410
-
1411
- Returns:
1412
- List of annotation dicts for visualization (empty if no tracks)
1413
- timestamp: Seconds from video start
1414
- source_id: Camera identifier
1415
- """
1416
- self._frame_count += 1
1417
- frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
1418
-
1419
- # ----- Phase 3: Detection -----
1420
- raw_detections = self.detector.predict(frame)
1421
-
1422
- if len(raw_detections) == 0:
1423
- return []
1424
-
1425
- # ----- Phase 4: TAI Filter -----
1426
- # Audit #8: wrap in try/except — active_tracks API varies across BoxMOT versions
1427
- tracked_boxes = np.empty((0, 4))
1428
- try:
1429
- if hasattr(self.tracker, 'active_tracks') and len(self.tracker.active_tracks) > 0:
1430
- tracked_boxes = np.array([t.xyxy for t in self.tracker.active_tracks])
1431
- except (AttributeError, TypeError):
1432
- pass # Graceful fallback — TAI filter runs without tracked boxes
1433
-
1434
- clean_detections = apply_tai_heuristic(
1435
- raw_detections, tracked_boxes, ioa_threshold=0.4
1436
- )
1437
-
1438
- # ----- Min-area filter: remove tiny background objects -----
1439
- # Construction equipment from aerial/drone footage has a minimum
1440
- # pixel footprint. Distant vehicles (like the background blue truck)
1441
- # are too small to be real tracked equipment.
1442
- if len(clean_detections) > 0:
1443
- frame_area = frame.shape[0] * frame.shape[1]
1444
- min_area_ratio = 0.003 # 0.3% of frame area (~6200px at 1080p)
1445
- areas = (clean_detections[:, 2] - clean_detections[:, 0]) * \
1446
- (clean_detections[:, 3] - clean_detections[:, 1])
1447
- area_mask = areas >= (frame_area * min_area_ratio)
1448
- clean_detections = clean_detections[area_mask]
1449
-
1450
- # ----- Phase 4: BoT-SORT Tracking -----
1451
- tracked_objects = self.tracker.update(clean_detections, frame)
1452
-
1453
- if self._frame_count % 30 == 0:
1454
- logger.info(f"Frame {frame_id}: Dets={len(raw_detections)}, Clean={len(clean_detections)}, Tracked={len(tracked_objects)}")
1455
-
1456
- # ----- Phase 5: Re-ID — detect lost tracks -----
1457
- current_track_ids = set()
1458
- if len(tracked_objects) > 0:
1459
- current_track_ids = {int(t[4]) for t in tracked_objects}
1460
-
1461
- lost_ids = self._prev_track_ids - current_track_ids
1462
- new_track_ids = current_track_ids - self._prev_track_ids
1463
-
1464
- for lost_id in lost_ids:
1465
- self.reid_gallery.move_to_gallery(lost_id, frame_id)
1466
- self.motion_detector.remove_track(f"pending_{lost_id}")
1467
- self.activity_classifier.remove_track(f"pending_{lost_id}")
1468
- self.video_classifier.remove_track(f"pending_{lost_id}")
1469
- self._track_first_seen.pop(lost_id, None)
1470
-
1471
- # ----- Process each tracked object -----
1472
- annotations = []
1473
- for track in tracked_objects:
1474
- bbox = track[:4]
1475
- track_id = int(track[4])
1476
- conf = float(track[5])
1477
- class_id = int(track[6])
1478
-
1479
- # Record first-seen frame for deferred ID creation
1480
- if track_id not in self._track_first_seen:
1481
- self._track_first_seen[track_id] = frame_id
1482
-
1483
- track_age = frame_id - self._track_first_seen[track_id]
1484
- equip_id = self.id_generator.get_existing(track_id)
1485
-
1486
- # ----- Phase 5: Re-ID + Equipment ID assignment -----
1487
- if track_id in new_track_ids or self._frame_count <= 1:
1488
- # New track — attempt Re-ID immediately
1489
- embedding = self.reid_gallery.extract_embedding(frame, bbox)
1490
- color_hist = self.reid_gallery._extract_color_histogram(frame, bbox)
1491
- matched_id = None
1492
-
1493
- if equip_id is None and embedding is not None:
1494
- matched_id, sim = self.reid_gallery.query(
1495
- embedding, class_id, frame_id,
1496
- query_bbox=bbox,
1497
- frame_width=frame.shape[1],
1498
- frame_height=frame.shape[0],
1499
- query_color_hist=color_hist
1500
- )
1501
- if matched_id is not None:
1502
- if self.id_generator.is_active(matched_id, exclude_track=track_id, active_track_ids=current_track_ids):
1503
- logger.info(f"Re-ID rejected: {matched_id} already active on another track")
1504
- matched_id = None
1505
- else:
1506
- equip_id = matched_id
1507
- self.id_generator.reassign(track_id, equip_id)
1508
- locked = self.id_generator.get_locked_class(equip_id)
1509
- if locked is not None:
1510
- class_id = locked
1511
-
1512
- if embedding is not None:
1513
- self.reid_gallery.update_active(
1514
- track_id, embedding, equip_id, class_id,
1515
- bbox, frame_id
1516
- )
1517
- if color_hist is not None and track_id in self.reid_gallery._active_embeddings:
1518
- self.reid_gallery._active_embeddings[track_id]['color_hist'] = color_hist
1519
-
1520
- if equip_id is not None:
1521
- locked_class = self.id_generator.get_locked_class(equip_id)
1522
- if locked_class is not None:
1523
- class_id = locked_class
1524
- else:
1525
- continue # Pending — skip annotation
1526
-
1527
- elif equip_id is None:
1528
- # Existing track, no equipment ID yet — check if mature enough
1529
- if track_age >= self.MIN_TRACK_FRAMES:
1530
- embedding = self.reid_gallery.extract_embedding(frame, bbox)
1531
- color_hist = self.reid_gallery._extract_color_histogram(frame, bbox)
1532
- matched_id = None
1533
-
1534
- if embedding is not None:
1535
- matched_id, sim = self.reid_gallery.query(
1536
- embedding, class_id, frame_id,
1537
- query_bbox=bbox,
1538
- frame_width=frame.shape[1],
1539
- frame_height=frame.shape[0],
1540
- query_color_hist=color_hist
1541
- )
1542
- if matched_id is not None:
1543
- if self.id_generator.is_active(matched_id, exclude_track=track_id, active_track_ids=current_track_ids):
1544
- logger.info(f"Re-ID rejected: {matched_id} already active on another track")
1545
- matched_id = None
1546
- else:
1547
- equip_id = matched_id
1548
- self.id_generator.reassign(track_id, equip_id)
1549
-
1550
- if equip_id is None:
1551
- equip_id = self.id_generator.get_or_create(track_id, class_id, conf=conf)
1552
- logger.info(f"Track {track_id} confirmed after {track_age} frames → {equip_id}")
1553
-
1554
- locked_class = self.id_generator.get_locked_class(equip_id)
1555
- if locked_class is not None:
1556
- class_id = locked_class
1557
-
1558
- if embedding is not None:
1559
- self.reid_gallery.update_active(
1560
- track_id, embedding, equip_id, class_id,
1561
- bbox, frame_id
1562
- )
1563
- if color_hist is not None and track_id in self.reid_gallery._active_embeddings:
1564
- self.reid_gallery._active_embeddings[track_id]['color_hist'] = color_hist
1565
- else:
1566
- if self._frame_count % 10 == 0:
1567
- embedding = self.reid_gallery.extract_embedding(frame, bbox)
1568
- if embedding is not None:
1569
- self.reid_gallery.update_active(
1570
- track_id, embedding, None, class_id,
1571
- bbox, frame_id
1572
- )
1573
- continue # Still pending
1574
-
1575
- else:
1576
- # Existing track WITH equipment ID — normal path
1577
- if not self.id_generator.is_class_locked(equip_id):
1578
- self.id_generator.vote_class(track_id, equip_id, class_id, conf)
1579
- equip_id = self.id_generator.get_existing(track_id)
1580
- locked_class = self.id_generator.get_locked_class(equip_id)
1581
- if locked_class is not None:
1582
- class_id = locked_class
1583
- if self._frame_count % 30 == 0:
1584
- embedding = self.reid_gallery.extract_embedding(frame, bbox)
1585
- if embedding is not None:
1586
- self.reid_gallery.update_active(
1587
- track_id, embedding, equip_id, class_id,
1588
- bbox, frame_id
1589
- )
1590
- color_hist = self.reid_gallery._extract_color_histogram(frame, bbox)
1591
- if color_hist is not None and track_id in self.reid_gallery._active_embeddings:
1592
- self.reid_gallery._active_embeddings[track_id]['color_hist'] = color_hist
1593
-
1594
- stable_id = equip_id if equip_id is not None else f"pending_{track_id}"
1595
-
1596
- # ----- Phase 6: Articulated Motion -----
1597
- motion_result = self.motion_detector.detect(
1598
- frame_gray, stable_id, bbox
1599
- )
1600
-
1601
- # ----- Phase 7: Pass through raw motion state -----
1602
- # Phase 10 owns ALL activity classification using spatial context.
1603
- # Phase 7 only records the motion detector's ACTIVE/INACTIVE decision
1604
- # and motion_source — no activity sub-labels assigned here.
1605
- equip_class = self._class_name(class_id)
1606
- is_active = motion_result['is_active']
1607
-
1608
- # Collect annotation — state/activity assigned by Phase 10
1609
- annotations.append({
1610
- 'bbox': bbox,
1611
- 'equip_id': equip_id,
1612
- 'stable_id': stable_id,
1613
- 'class_name': equip_class,
1614
- 'conf': conf,
1615
- 'is_motion_active': is_active, # raw from motion detector
1616
- 'motion_source': motion_result['motion_source'],
1617
- 'cell_motions': motion_result['cell_motions'], # 3x3 grid for bucket height
1618
- 'state': '', # Phase 10 assigns
1619
- 'activity': '', # Phase 10 assigns
1620
- 'frame_id': frame_id,
1621
- 'timestamp': f"{int(timestamp // 3600):02d}:{int((timestamp % 3600) // 60):02d}:{timestamp % 60:06.3f}",
1622
- 'source_id': source_id,
1623
- 'track_id': track_id, # needed for video classifier lookup
1624
- })
1625
-
1626
- # Feed crop to video activity classifier buffer
1627
- if self.video_classifier.is_loaded:
1628
- x1, y1, x2, y2 = [int(v) for v in bbox[:4]]
1629
- x1, y1 = max(0, x1), max(0, y1)
1630
- x2 = min(frame.shape[1], x2)
1631
- y2 = min(frame.shape[0], y2)
1632
- crop = frame[y1:y2, x1:x2]
1633
- self.video_classifier.feed_crop(stable_id, crop)
1634
-
1635
- # ----- Phase 10: Activity Classification -----
1636
- # Single source of truth for ALL state + activity labels.
1637
- #
1638
- # Uses two signals from the 3x3 optical flow grid:
1639
- # 1. Bucket height: flow concentrated in TOP rows = bucket elevated (DUMPING)
1640
- # flow concentrated in BOTTOM rows = bucket low (DIGGING)
1641
- # 2. Motion source: whole_body = machine translating (LOADING/swinging)
1642
- # partial = only part moving (bucket work)
1643
- #
1644
- # Decision table for articulated equipment:
1645
- # INACTIVE → WAITING
1646
- # ACTIVE + whole_body → LOADING (swinging/translating)
1647
- # ACTIVE + partial + flow in top rows → DUMPING (bucket elevated)
1648
- # ACTIVE + partial + flow in bottom rows → DIGGING (bucket low, scooping)
1649
- #
1650
- # Dump trucks: classified via adjacent active articulated equipment.
1651
-
1652
- ARTICULATED = {'excavator', 'wheel_loader', 'bulldozer'}
1653
-
1654
- def bucket_height_score(cell_motions):
1655
- """
1656
- Compute where the flow is concentrated vertically in the 3x3 grid.
1657
- Returns a score: positive = top rows (bucket high), negative = bottom rows (bucket low).
1658
-
1659
- Grid layout (3x3):
1660
- row 0 = top of bbox (bucket elevated / dumping position)
1661
- row 1 = middle
1662
- row 2 = bottom of bbox (bucket low / digging position)
1663
- """
1664
- if not cell_motions or len(cell_motions) < 3:
1665
- return 0.0
1666
- # Sum flow magnitude across each row (all 3 columns)
1667
- top_flow = sum(cell_motions[0]) # row 0
1668
- mid_flow = sum(cell_motions[1]) # row 1
1669
- bottom_flow = sum(cell_motions[2]) # row 2
1670
- total = top_flow + mid_flow + bottom_flow
1671
- if total < 0.01:
1672
- return 0.0
1673
- # Score: +1 = all flow in top, -1 = all flow in bottom
1674
- return (top_flow - bottom_flow) / total
1675
-
1676
- def bboxes_overlap(a, b):
1677
- x_overlap = min(a[2], b[2]) - max(a[0], b[0])
1678
- y_overlap = min(a[3], b[3]) - max(a[1], b[1])
1679
- return x_overlap > 0 and y_overlap > 0
1680
-
1681
- # --- Pass 1: Classify articulated equipment ---
1682
- # If X3D-S video classifier is loaded, use it (Option B).
1683
- # Otherwise, fall back to bucket height heuristic (Option A).
1684
- use_video_classifier = self.video_classifier.is_loaded
1685
-
1686
- # Calculate smoothing window dynamically (0.50 seconds to reduce lag)
1687
- window_size = max(10, int(fps * 0.50))
1688
-
1689
- for ann in annotations:
1690
- if ann is None or ann['class_name'] not in ARTICULATED:
1691
- continue
1692
-
1693
- # Get video classifier output if available and motion is active
1694
- if use_video_classifier and ann['is_motion_active']:
1695
- activity, conf = self.video_classifier.classify(ann['stable_id'])
1696
- if activity is not None:
1697
- ann['state'] = 'ACTIVE'
1698
- ann['activity'] = activity
1699
- ann['activity_conf'] = conf
1700
- continue
1701
-
1702
- # Fallback: rule-based Option A
1703
- raw_activity = 'WAITING'
1704
- if ann['is_motion_active']:
1705
- score = bucket_height_score(ann['cell_motions'])
1706
- if ann['motion_source'] == 'whole_body':
1707
- if score < -0.15:
1708
- raw_activity = 'DIGGING'
1709
- else:
1710
- raw_activity = 'LOADING'
1711
- else:
1712
- if score > 0.1:
1713
- raw_activity = 'DUMPING'
1714
- else:
1715
- raw_activity = 'DIGGING'
1716
-
1717
- # Apply temporal smoothing for Option A using equip_id
1718
- eid = ann['equip_id']
1719
- if eid not in self._activity_history:
1720
- self._activity_history[eid] = deque(maxlen=window_size)
1721
-
1722
- if self._activity_history[eid].maxlen != window_size:
1723
- old_items = list(self._activity_history[eid])
1724
- self._activity_history[eid] = deque(old_items[-window_size:], maxlen=window_size)
1725
-
1726
- self._activity_history[eid].append(raw_activity)
1727
-
1728
- # Majority vote (protect against plurality vote splitting active states)
1729
- from collections import Counter
1730
- counts = Counter(self._activity_history[eid])
1731
- waiting_votes = counts.get('WAITING', 0)
1732
- active_votes = sum(c for s, c in counts.items() if s != 'WAITING')
1733
-
1734
- if waiting_votes > active_votes:
1735
- ann['activity'] = 'WAITING'
1736
- ann['state'] = 'INACTIVE'
1737
- else:
1738
- active_only = {s: c for s, c in counts.items() if s != 'WAITING'}
1739
- ann['activity'] = max(active_only.items(), key=lambda x: x[1])[0] if active_only else 'LOADING'
1740
- ann['state'] = 'ACTIVE'
1741
-
1742
- # --- Pass 2: Classify dump trucks via adjacent active articulated ---
1743
- dt_annotations = [a for a in annotations if a is not None and a['class_name'] == 'dump_truck']
1744
- active_articulateds = [
1745
- a for a in annotations
1746
- if a is not None
1747
- and a['state'] == 'ACTIVE'
1748
- and a['class_name'] in ARTICULATED
1749
- ]
1750
- for dt in dt_annotations:
1751
- d_bbox = dt['bbox'][:4]
1752
- near_loader = False
1753
- for loader in active_articulateds:
1754
- if bboxes_overlap(d_bbox, loader['bbox'][:4]):
1755
- near_loader = True
1756
- break
1757
-
1758
- if near_loader:
1759
- dt['state'] = 'ACTIVE'
1760
- dt['activity'] = 'LOADING'
1761
- elif dt['is_motion_active']:
1762
- # Truck is moving — map to default ACTIVE (LOADING) since HAULING is not in the 4-activity schema!
1763
- dt['state'] = 'ACTIVE'
1764
- dt['activity'] = 'LOADING'
1765
- else:
1766
- dt['state'] = 'INACTIVE'
1767
- dt['activity'] = 'WAITING'
1768
-
1769
- # --- Pass 3: Default for any remaining equipment (cranes, etc.) ---
1770
- for ann in annotations:
1771
- if ann is None or ann['state'] != '':
1772
- continue
1773
- if ann['is_motion_active']:
1774
- ann['state'] = 'ACTIVE'
1775
- ann['activity'] = 'LOADING'
1776
- else:
1777
- ann['state'] = 'INACTIVE'
1778
- ann['activity'] = 'WAITING'
1779
-
1780
- # Debug: log Phase 10 decisions once per second
1781
- if frame_id % 60 == 0:
1782
- for ann in annotations:
1783
- if ann is None:
1784
- continue
1785
- bh = bucket_height_score(ann.get('cell_motions', []))
1786
- src = 'x3d' if ann.get('activity_conf') else 'rules'
1787
- conf_str = f" ({ann['activity_conf']:.2f})" if ann.get('activity_conf') else ''
1788
- logger.info(
1789
- f"Phase10[{ann['equip_id']}]: class={ann['class_name']}, "
1790
- f"motion_active={ann['is_motion_active']}, "
1791
- f"motion_src={ann['motion_source']}, bucket_h={bh:+.2f}, "
1792
- f"→ {ann['state']} | {ann['activity']}{conf_str} [{src}]"
1793
- )
1794
-
1795
- # ----- Deduplicate equipment IDs -----
1796
- # If two tracks share the same equipment ID (e.g., WL-002 on both the
1797
- # real wheel loader and a false detection), keep only the higher-confidence
1798
- # annotation. The lower-confidence duplicate is dropped from display.
1799
- seen_ids = {}
1800
- for i, ann in enumerate(annotations):
1801
- eid = ann['equip_id']
1802
- if eid in seen_ids:
1803
- prev_idx = seen_ids[eid]
1804
- if ann['conf'] > annotations[prev_idx]['conf']:
1805
- annotations[prev_idx] = None # mark old for removal
1806
- seen_ids[eid] = i
1807
- else:
1808
- annotations[i] = None # mark new for removal
1809
- else:
1810
- seen_ids[eid] = i
1811
- annotations = [a for a in annotations if a is not None]
1812
-
1813
- # ----- Phase 11: Produce to Kafka (after spatial context + dedup) -----
1814
- for ann in annotations:
1815
- payload = {
1816
- 'frame_id': ann['frame_id'],
1817
- 'equipment_id': ann['equip_id'],
1818
- 'equipment_class': ann['class_name'],
1819
- 'timestamp': ann['timestamp'],
1820
- 'bbox': {
1821
- 'x1': float(ann['bbox'][0]), 'y1': float(ann['bbox'][1]),
1822
- 'x2': float(ann['bbox'][2]), 'y2': float(ann['bbox'][3])
1823
- },
1824
- 'detection_confidence': round(ann['conf'], 3),
1825
- 'utilization': {
1826
- 'current_state': ann['state'],
1827
- 'current_activity': ann['activity'],
1828
- 'motion_source': ann['motion_source']
1829
- },
1830
- 'source_id': ann['source_id'],
1831
- 'pipeline_timestamp': datetime.now(timezone.utc).isoformat()
1832
- }
1833
- self.telemetry_producer.produce(ann['equip_id'], payload)
1834
-
1835
- # Periodic flush
1836
- if self._frame_count % 30 == 0:
1837
- self.telemetry_producer.flush()
1838
-
1839
- # Periodic gallery cleanup — recycle expired phantom IDs (self-healing)
1840
- if self._frame_count % 300 == 0: # Every 5 seconds at 60fps
1841
- self.reid_gallery.cleanup(frame_id, id_generator=self.id_generator)
1842
-
1843
- # Cleanup stale activity history
1844
- active_known_eids = set(self.id_generator._equip_to_class.keys())
1845
- stale_hist = [eid for eid in self._activity_history if eid not in active_known_eids]
1846
- for eid in stale_hist:
1847
- del self._activity_history[eid]
1848
-
1849
- # Cleanup stale stable_id memories inside motion detector
1850
- if hasattr(self.motion_detector, '_prev_crops'):
1851
- stale_md = [eid for eid in self.motion_detector._prev_crops if not str(eid).startswith('pending_') and eid not in active_known_eids]
1852
- for eid in stale_md:
1853
- self.motion_detector.remove_track(eid)
1854
-
1855
- # Cleanup stale stable_id memories inside video classifier
1856
- if hasattr(self.video_classifier, '_crop_buffer'):
1857
- stale_vc = [eid for eid in self.video_classifier._crop_buffer if not str(eid).startswith('pending_') and eid not in active_known_eids]
1858
- for eid in stale_vc:
1859
- self.video_classifier.remove_track(eid)
1860
-
1861
- # Audit #2: update _prev_track_ids AFTER the loop so new_track_ids works next frame
1862
- self._prev_track_ids = current_track_ids
1863
-
1864
- return annotations
1865
-
1866
-
1867
- @staticmethod
1868
- def _class_name(class_id: int) -> str:
1869
- return CLASS_NAMES.get(int(class_id), 'unknown')
1870
-
1871
- @staticmethod
1872
- def draw_annotations(frame: np.ndarray, annotations: list,
1873
- frame_id: int, timestamp: float = 0.0,
1874
- fps: float = 0.0) -> np.ndarray:
1875
- """
1876
- Draw bounding boxes, labels, and overlay info on the frame.
1877
- Returns annotated frame copy.
1878
- """
1879
- vis = frame.copy()
1880
-
1881
- # Color palette per class
1882
- COLORS = {
1883
- 'excavator': (0, 200, 255), # Orange
1884
- 'dump_truck': (255, 180, 0), # Cyan-blue
1885
- 'bulldozer': (0, 255, 100), # Green
1886
- 'wheel_loader': (0, 255, 255), # Yellow
1887
- 'mobile_crane': (255, 0, 150), # Magenta
1888
- 'tower_crane': (150, 100, 255), # Purple
1889
- 'roller_compactor':(255, 100, 50), # Red-orange
1890
- 'cement_mixer': (100, 255, 200), # Teal
1891
- 'unknown': (200, 200, 200), # Gray
1892
- }
1893
-
1894
- for ann in annotations:
1895
- x1, y1, x2, y2 = [int(v) for v in ann['bbox'][:4]]
1896
- color = COLORS.get(ann['class_name'], (200, 200, 200))
1897
- state_color = (0, 255, 0) if ann['state'] == 'ACTIVE' else (0, 0, 255)
1898
-
1899
- # Draw bbox
1900
- cv2.rectangle(vis, (x1, y1), (x2, y2), color, 2)
1901
-
1902
- # State indicator bar (top of bbox)
1903
- cv2.rectangle(vis, (x1, y1 - 4), (x2, y1), state_color, -1)
1904
-
1905
- # Label background
1906
- label = f"{ann['equip_id']} | {ann['class_name']} | {ann['conf']:.0%}"
1907
- (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
1908
- cv2.rectangle(vis, (x1, y1 - 24), (x1 + tw + 6, y1 - 4), color, -1)
1909
- cv2.putText(vis, label, (x1 + 3, y1 - 9),
1910
- cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
1911
-
1912
- # Activity label below bbox
1913
- act_label = f"{ann['state']} | {ann['activity']}"
1914
- cv2.putText(vis, act_label, (x1, y2 + 16),
1915
- cv2.FONT_HERSHEY_SIMPLEX, 0.45, state_color, 1, cv2.LINE_AA)
1916
-
1917
- # HUD overlay with video timestamp
1918
- mins = int(timestamp // 60)
1919
- secs = timestamp % 60
1920
- fps_str = f"{fps:.1f}" if fps >= 0 else "..."
1921
- hud = f"{mins:02d}:{secs:05.2f} | Tracked: {len(annotations)} | FPS: {fps_str}"
1922
- cv2.putText(vis, hud, (10, 25),
1923
- cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2, cv2.LINE_AA)
1924
-
1925
- return vis
1926
-
1927
- def shutdown(self):
1928
- """Clean shutdown."""
1929
- self.telemetry_producer.flush()
1930
- logger.info("Pipeline shutdown complete.")
1931
-
1932
-
1933
- # =============================================================================
1934
- # Entry Point
1935
- # =============================================================================
1936
- def main():
1937
- logger.info("=" * 60)
1938
- logger.info(" Construction Equipment Monitor - CV Inference")
1939
- logger.info("=" * 60)
1940
-
1941
- # cv2 is now imported at module level
1942
-
1943
- config = {
1944
- 'kafka_servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'kafka:9092'),
1945
- 'kafka_topic': os.getenv('KAFKA_TOPIC_RAW', 'equipment.telemetry.raw'),
1946
- 'video_source': os.getenv('VIDEO_SOURCE', '/data/test_video.mp4'),
1947
- 'device': os.getenv('DEVICE', 'cuda'),
1948
- 'motion_threshold': float(os.getenv('MOTION_THRESHOLD', '0.5')), # Audit #3: was 2.0
1949
- 'frame_skip': int(os.getenv('FRAME_SKIP', '1')),
1950
- 'target_width': int(os.getenv('TARGET_WIDTH', '960')),
1951
- 'confidence_threshold': float(os.getenv('CONFIDENCE_THRESHOLD', '0.50')),
1952
- 'reid_threshold': float(os.getenv('REID_THRESHOLD', '0.85')), # Conservative: Re-ID only fires after BoT-SORT drops track (>20s lost)
1953
- 'reid_gallery_ttl': int(os.getenv('REID_GALLERY_TTL', '18000')), # 5min at 60fps — vehicles can leave and return
1954
- 'weights_path': os.path.join(os.getenv('MODEL_PATH', '/models'), 'rfdetr_construction.pth'),
1955
- }
1956
-
1957
- # Initialize pipeline
1958
- pipeline = InferencePipeline(config)
1959
- pipeline.initialize()
1960
-
1961
- # Initialize video ingestion
1962
- from ingestion import VideoIngestionService
1963
- ingestion = VideoIngestionService(
1964
- source=config['video_source'],
1965
- frame_skip=config['frame_skip'],
1966
- target_width=config['target_width'],
1967
- queue_maxsize=30
1968
- )
1969
-
1970
- # Graceful shutdown handler
1971
- shutdown_flag = False
1972
- def signal_handler(sig, frame):
1973
- nonlocal shutdown_flag
1974
- logger.info("Shutdown signal received...")
1975
- shutdown_flag = True
1976
- signal.signal(signal.SIGINT, signal_handler)
1977
- signal.signal(signal.SIGTERM, signal_handler)
1978
-
1979
- # Initialize video writer for annotated output
1980
- save_video = os.getenv('SAVE_ANNOTATED', 'true').lower() == 'true'
1981
- video_writer = None
1982
- output_path = '/data/output_annotated.mp4'
1983
-
1984
- # Start processing
1985
- ingestion.start()
1986
- logger.info("Processing started. Press Ctrl+C to stop.")
1987
-
1988
- fps_counter = 0
1989
- fps_start = time.time()
1990
- current_fps = -1.0 # Sentinel: -1 means "not yet computed"
1991
-
1992
- while not shutdown_flag:
1993
- packet = ingestion.get_frame(timeout=2.0)
1994
- if packet is None:
1995
- logger.info("End of video stream.")
1996
- break
1997
-
1998
- annotations = pipeline.process_frame(
1999
- frame=packet.frame,
2000
- frame_id=packet.frame_id,
2001
- timestamp=packet.timestamp,
2002
- source_id=packet.source_id,
2003
- fps=getattr(ingestion, '_src_fps', 30.0)
2004
- )
2005
-
2006
- # Draw annotations and write to output video
2007
- if save_video:
2008
- if annotations:
2009
- vis_frame = InferencePipeline.draw_annotations(
2010
- packet.frame, annotations, packet.frame_id,
2011
- timestamp=packet.timestamp, fps=current_fps
2012
- )
2013
- else:
2014
- # Still write unannotated frames for smooth playback
2015
- vis_frame = packet.frame.copy()
2016
- mins = int(packet.timestamp // 60)
2017
- secs = packet.timestamp % 60
2018
- fps_str = f"{current_fps:.1f}" if current_fps >= 0 else "..."
2019
- hud = f"{mins:02d}:{secs:05.2f} | Tracked: 0 | FPS: {fps_str}"
2020
- cv2.putText(vis_frame, hud, (10, 25),
2021
- cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2, cv2.LINE_AA)
2022
- if video_writer is None:
2023
- h, w = vis_frame.shape[:2]
2024
- src_fps = ingestion._src_fps if hasattr(ingestion, '_src_fps') else 30.0
2025
- fourcc = cv2.VideoWriter_fourcc(*'mp4v')
2026
- video_writer = cv2.VideoWriter(output_path, fourcc, src_fps, (w, h))
2027
- logger.info(f"Saving annotated video to {output_path} @ {src_fps} FPS")
2028
- video_writer.write(vis_frame)
2029
-
2030
- # FPS tracking
2031
- fps_counter += 1
2032
- elapsed = time.time() - fps_start
2033
- # Update current_fps every frame so HUD never shows 0.0
2034
- # Require at least 2 frames to avoid misleading spikes on first frame
2035
- if elapsed > 0 and fps_counter >= 2:
2036
- current_fps = fps_counter / elapsed
2037
- if elapsed >= 5.0:
2038
- logger.info(
2039
- f"Pipeline FPS: {current_fps:.1f} | "
2040
- f"Frames: {packet.frame_id} | "
2041
- f"Dropped: {ingestion.frames_dropped}"
2042
- )
2043
- fps_counter = 0
2044
- fps_start = time.time()
2045
-
2046
- # Cleanup
2047
- if video_writer is not None:
2048
- video_writer.release()
2049
- logger.info(f"Annotated video saved to {output_path}")
2050
- ingestion.stop()
2051
- pipeline.shutdown()
2052
- logger.info("CV Inference service stopped.")
2053
 
 
2054
 
2055
- if __name__ == '__main__':
2056
- main()
 
1
+ ---
2
+ license: mit
3
+ language:
4
+ - en
5
+ tags:
6
+ - object-detection
7
+ - re-identification
8
+ - construction
9
+ - aerial-vision
10
+ - rf-detr
11
+ - dinov2
12
+ - osnet
13
+ - real-time
14
+ - tracking
15
+ pipeline_tag: object-detection
16
+ library_name: pytorch
17
+ datasets:
18
+ - roboflow
19
+ ---
20
 
21
+ # 🏗️ SiteSense — Model Weights
 
 
 
 
 
 
 
 
 
 
22
 
23
+ **Real-Time Construction Equipment Monitoring via Aerial Computer Vision**
 
 
 
 
 
 
 
24
 
25
+ [![GitHub](https://img.shields.io/badge/GitHub-Repository-181717?logo=github&logoColor=white)](https://github.com/Mahmoud-Zaafan/asdfqer)
26
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
27
+ [![Python 3.11](https://img.shields.io/badge/Python-3.11-3776AB?logo=python&logoColor=white)](https://python.org)
28
+ [![PyTorch 2.2+](https://img.shields.io/badge/PyTorch-2.2+-EE4C2C?logo=pytorch&logoColor=white)](https://pytorch.org)
29
 
30
+ ---
31
 
32
+ ## Overview
 
33
 
34
+ This repository hosts the trained model weights for [SiteSense](https://github.com/Mahmoud-Zaafan/asdfqer) — a real-time pipeline that **detects, tracks, identifies, and classifies the activity** of heavy construction equipment from drone/aerial video footage.
 
 
 
 
 
 
 
 
 
 
35
 
36
+ The system processes each frame through a multi-phase pipeline:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
 
38
+ ```
39
+ Video Frame → RF-DETR Detection → BoT-SORT Tracking → DINOv3 Re-ID → Activity Classification → Kafka Events
40
+ ```
41
 
42
+ ---
 
 
 
 
43
 
44
+ ## Model Weights
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
 
46
+ | File | Size | Architecture | Task | Training Data |
47
+ |:---|:---:|:---|:---|:---|
48
+ | `rfdetr_construction.pth` | 122 MB | RF-DETR (Real-time Foundation DETR) | 8-class object detection | Custom aerial construction dataset (Roboflow) |
49
+ | `dinov3_reid_head.pth` | 5.4 MB | Linear projection head (1536→256→128) | Equipment re-identification | Contrastive pairs from tracked equipment |
50
+ | `osnet_x0_25_msmt17.pt` | 2.9 MB | OSNet x0.25 | Appearance-based ReID for BoT-SORT | MSMT17 (pretrained) |
51
 
52
+ > **Note:** The DINOv3 ViT-B/16 backbone (~327 MB) is **not included** here. It is auto-downloaded from [facebook/dinov3-vitb16-pretrain-lvd1689m](https://huggingface.co/facebook/dinov3-vitb16-pretrain-lvd1689m) on first run using your `HF_TOKEN`.
 
 
 
 
 
 
53
 
54
+ ---
 
 
55
 
56
+ ## Detection Classes
 
57
 
58
+ The RF-DETR detector is fine-tuned to recognize **8 classes** of construction equipment from aerial perspectives:
59
 
60
+ | ID | Class | ID | Class |
61
+ |:---:|:---|:---:|:---|
62
+ | 0 | Excavator | 4 | Mobile Crane |
63
+ | 1 | Dump Truck | 5 | Tower Crane |
64
+ | 2 | Bulldozer | 6 | Roller Compactor |
65
+ | 3 | Wheel Loader | 7 | Cement Mixer |
 
 
 
 
66
 
67
+ ---
 
 
 
68
 
69
+ ## Training Results
 
 
70
 
71
+ ### RF-DETR Detector
 
 
 
 
 
72
 
73
+ | Metric | Value |
74
+ |:---|:---:|
75
+ | **mAP@50** | 0.8340 |
76
+ | **mAP@50:95** | 0.7607 |
77
+ | **F1 Score** | 0.8859 |
78
+ | **Precision** | 0.8666 |
79
+ | **Recall** | 0.9061 |
80
+ | Resolution | 560×560 |
81
+ | Epochs | 70 |
82
 
83
+ ### DINOv3 Re-ID Projection Head
84
 
85
+ | Metric | Value |
86
+ |:---|:---:|
87
+ | **Contrastive Loss** | 0.0482 |
88
+ | **Accuracy** | 96.8% |
89
+ | Embedding Dim | 128-d L2-normalized |
90
+ | Training Pairs | ~12,000 positive pairs |
91
 
92
+ ---
 
 
93
 
94
+ ## Quick Start
 
 
 
 
 
 
95
 
96
+ ### Option A: Download All Weights (Recommended)
 
 
97
 
98
+ ```bash
99
+ pip install huggingface_hub
100
+ huggingface-cli download Zaafan/sitesense-weights --local-dir models/
101
+ ```
 
 
 
 
 
102
 
103
+ ### Option B: Python API
 
 
 
104
 
105
+ ```python
106
+ from huggingface_hub import hf_hub_download
 
107
 
108
+ # Download individual weights
109
+ hf_hub_download(repo_id="Zaafan/sitesense-weights", filename="rfdetr_construction.pth", local_dir="models/")
110
+ hf_hub_download(repo_id="Zaafan/sitesense-weights", filename="dinov3_reid_head.pth", local_dir="models/")
111
+ hf_hub_download(repo_id="Zaafan/sitesense-weights", filename="osnet_x0_25_msmt17.pt", local_dir="models/")
112
+ ```
113
 
114
+ ### Option C: Auto-Download (Zero Setup)
 
 
 
 
 
115
 
116
+ The SiteSense pipeline automatically downloads missing weights on first run:
 
 
 
117
 
118
+ ```python
119
+ # In services/cv-inference/main.py — resolve_weights() handles this transparently
120
+ weights_path = resolve_weights('rfdetr_construction.pth') # local first, HF fallback
121
+ ```
122
 
123
+ ---
 
 
 
124
 
125
+ ## Usage with SiteSense Pipeline
 
 
126
 
127
+ ```bash
128
+ # 1. Clone the repository
129
+ git clone https://github.com/Mahmoud-Zaafan/asdfqer.git
130
+ cd asdfqer
 
 
131
 
132
+ # 2. Download weights
133
+ huggingface-cli download Zaafan/sitesense-weights --local-dir models/
134
 
135
+ # 3. Configure environment
136
+ cp .env.example .env
 
 
 
 
137
 
138
+ # 4. Launch infrastructure + run pipeline
139
+ docker compose up --build
140
+ docker compose --profile pipeline up cv-inference
141
+ ```
142
 
143
+ ---
 
144
 
145
+ ## Citation
146
 
147
+ If you use these weights in your research or projects, please cite:
 
 
 
148
 
149
+ ```bibtex
150
+ @misc{sitesense2025,
151
+ author = {Mahmoud Zaafan},
152
+ title = {SiteSense: Real-Time Construction Equipment Monitoring via Aerial Computer Vision},
153
+ year = {2025},
154
+ url = {https://github.com/Mahmoud-Zaafan/asdfqer}
155
+ }
156
+ ```
157
 
158
+ ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
 
160
+ ## License
161
 
162
+ All weights are released under the [MIT License](LICENSE).