Update README.md
Browse files
README.md
CHANGED
|
@@ -1,2056 +1,162 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
-
|
| 8 |
-
1. Video Ingestion → FramePacket
|
| 9 |
-
2. (Optional) Zero-DCE low-light enhancement
|
| 10 |
-
3. RF-DETR Detection → raw detections [x1,y1,x2,y2,conf,cls]
|
| 11 |
-
4. TAI Heuristic Filter → clean detections (suppress fragments)
|
| 12 |
-
5. BoT-SORT Tracking → tracked objects with persistent IDs
|
| 13 |
-
6. DINOv3 Re-ID Gallery → re-identify lost tracks
|
| 14 |
-
7. Grid Decomposition + Farneback → articulated motion detection
|
| 15 |
-
8. Rule-based Activity Classifier → DIGGING/LOADING/DUMPING/WAITING
|
| 16 |
-
9. Kafka Producer → equipment.telemetry.raw
|
| 17 |
-
"""
|
| 18 |
|
| 19 |
-
|
| 20 |
-
import sys
|
| 21 |
-
import json
|
| 22 |
-
import time
|
| 23 |
-
import logging
|
| 24 |
-
import signal
|
| 25 |
-
from datetime import datetime, timezone
|
| 26 |
-
from pathlib import Path
|
| 27 |
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
|
|
|
| 31 |
|
|
|
|
| 32 |
|
| 33 |
-
#
|
| 34 |
-
HF_REPO_ID = "Zaafan/sitesense-weights"
|
| 35 |
|
| 36 |
-
|
| 37 |
-
"""Return the path to a model weight file.
|
| 38 |
-
|
| 39 |
-
Priority:
|
| 40 |
-
1. Local file at <local_dir>/<filename> (fastest — no network)
|
| 41 |
-
2. Auto-download from Hugging Face Hub (fallback)
|
| 42 |
-
"""
|
| 43 |
-
local_path = os.path.join(local_dir, filename)
|
| 44 |
-
if os.path.exists(local_path):
|
| 45 |
-
logger.info(f"Found local weights: {local_path}")
|
| 46 |
-
return local_path
|
| 47 |
|
| 48 |
-
|
| 49 |
-
try:
|
| 50 |
-
from huggingface_hub import hf_hub_download
|
| 51 |
-
downloaded = hf_hub_download(
|
| 52 |
-
repo_id=HF_REPO_ID,
|
| 53 |
-
filename=filename,
|
| 54 |
-
local_dir=local_dir,
|
| 55 |
-
)
|
| 56 |
-
logger.info(f"Downloaded weights to: {downloaded}")
|
| 57 |
-
return downloaded
|
| 58 |
-
except Exception as e:
|
| 59 |
-
logger.error(f"Failed to download {filename} from HuggingFace: {e}")
|
| 60 |
-
raise FileNotFoundError(
|
| 61 |
-
f"Model weights '{filename}' not found locally at {local_path} "
|
| 62 |
-
f"and could not be downloaded from {HF_REPO_ID}. "
|
| 63 |
-
f"Run: huggingface-cli download {HF_REPO_ID} --local-dir {local_dir}"
|
| 64 |
-
)
|
| 65 |
-
from collections import deque
|
| 66 |
-
from PIL import Image # Moved to top level (audit #11)
|
| 67 |
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
|
| 72 |
-
|
| 73 |
-
level=logging.INFO,
|
| 74 |
-
format='%(asctime)s [%(name)s] %(levelname)s: %(message)s'
|
| 75 |
-
)
|
| 76 |
-
logger = logging.getLogger('cv-inference')
|
| 77 |
|
| 78 |
-
#
|
| 79 |
-
CLASS_NAMES = {
|
| 80 |
-
0: 'excavator', 1: 'dump_truck', 2: 'bulldozer', 3: 'wheel_loader',
|
| 81 |
-
4: 'mobile_crane', 5: 'tower_crane', 6: 'roller_compactor', 7: 'cement_mixer',
|
| 82 |
-
}
|
| 83 |
-
CLASS_PREFIXES = {
|
| 84 |
-
0: 'EX', 1: 'DT', 2: 'BD', 3: 'WL',
|
| 85 |
-
4: 'MC', 5: 'TC', 6: 'RC', 7: 'CM',
|
| 86 |
-
}
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
# =============================================================================
|
| 90 |
-
# RF-DETR Detector Wrapper
|
| 91 |
-
# =============================================================================
|
| 92 |
-
class RFDETRDetector:
|
| 93 |
-
"""
|
| 94 |
-
Wraps the fine-tuned RF-DETR model for inference.
|
| 95 |
-
Loads checkpoint_best_regular.pth trained on our 8-class dataset.
|
| 96 |
-
"""
|
| 97 |
-
|
| 98 |
-
def __init__(self, weights_path: str, num_classes: int = 8,
|
| 99 |
-
confidence_threshold: float = 0.35, device: str = 'cuda'):
|
| 100 |
-
self.confidence_threshold = confidence_threshold
|
| 101 |
-
self.device = device if torch.cuda.is_available() else 'cpu'
|
| 102 |
-
self.model = None
|
| 103 |
-
self.weights_path = weights_path
|
| 104 |
-
|
| 105 |
-
def load(self):
|
| 106 |
-
"""Load the fine-tuned RF-DETR model."""
|
| 107 |
-
from rfdetr import RFDETRBase
|
| 108 |
-
# In rfdetr, the weights path is passed as `pretrain_weights`
|
| 109 |
-
self.model = RFDETRBase(num_classes=8, pretrain_weights=self.weights_path)
|
| 110 |
-
# Optimize model for faster inference (torch.compile / fused ops)
|
| 111 |
-
if hasattr(self.model, 'optimize_for_inference'):
|
| 112 |
-
self.model.optimize_for_inference()
|
| 113 |
-
logger.info("RF-DETR inference optimization applied")
|
| 114 |
-
logger.info(f"RF-DETR loaded from {self.weights_path} on {self.device}")
|
| 115 |
-
|
| 116 |
-
def predict(self, frame: np.ndarray) -> np.ndarray:
|
| 117 |
-
"""
|
| 118 |
-
Run detection on a single frame.
|
| 119 |
-
|
| 120 |
-
Args:
|
| 121 |
-
frame: BGR image (H, W, 3)
|
| 122 |
-
|
| 123 |
-
Returns:
|
| 124 |
-
(N, 6) array of [x1, y1, x2, y2, confidence, class_id]
|
| 125 |
-
"""
|
| 126 |
-
if self.model is None:
|
| 127 |
-
return np.empty((0, 6))
|
| 128 |
-
|
| 129 |
-
detections = self.model.predict(frame, threshold=self.confidence_threshold)
|
| 130 |
-
|
| 131 |
-
# RF-DETR returns a Supervision Detections object
|
| 132 |
-
# with .xyxy (N,4), .confidence (N,), .class_id (N,)
|
| 133 |
-
if detections is None or len(detections) == 0:
|
| 134 |
-
return np.empty((0, 6))
|
| 135 |
-
|
| 136 |
-
boxes = detections.xyxy # (N, 4)
|
| 137 |
-
confs = detections.confidence.reshape(-1, 1) # (N, 1)
|
| 138 |
-
clses = detections.class_id.reshape(-1, 1) # (N, 1)
|
| 139 |
-
|
| 140 |
-
return np.hstack([boxes, confs, clses]).astype(np.float32)
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
# =============================================================================
|
| 144 |
-
# DINOv3 Re-ID Gallery (Persistent Identity Across Occlusions)
|
| 145 |
-
# =============================================================================
|
| 146 |
-
class DINOv3ReIDGallery:
|
| 147 |
-
"""
|
| 148 |
-
Uses DINOv3 ViT-B/16 to extract visual embeddings from equipment crops.
|
| 149 |
-
Maintains a gallery of lost tracks to re-identify them when they reappear.
|
| 150 |
-
|
| 151 |
-
This solves the critical problem: when two identical yellow haul trucks
|
| 152 |
-
drive behind a dirt mound, the tracker loses their IDs. This gallery
|
| 153 |
-
matches them back to their original identity using visual fingerprints.
|
| 154 |
-
"""
|
| 155 |
-
|
| 156 |
-
def __init__(self, similarity_threshold: float = 0.95,
|
| 157 |
-
gallery_ttl_frames: int = 1800, device: str = 'cuda',
|
| 158 |
-
color_threshold: float = 0.40,
|
| 159 |
-
spatial_gate_ratio: float = 0.40):
|
| 160 |
-
self.color_threshold = color_threshold
|
| 161 |
-
self.spatial_gate_ratio = spatial_gate_ratio
|
| 162 |
-
self.similarity_threshold = similarity_threshold
|
| 163 |
-
self.gallery_ttl_frames = gallery_ttl_frames
|
| 164 |
-
self.device = device if torch.cuda.is_available() else 'cpu'
|
| 165 |
-
|
| 166 |
-
self.model = None
|
| 167 |
-
self.processor = None
|
| 168 |
-
self.projection = None
|
| 169 |
-
|
| 170 |
-
# Gallery: track_id -> {embedding, equipment_id, class_id, last_frame, bbox}
|
| 171 |
-
self._gallery = {}
|
| 172 |
-
# Active tracks: track_id -> embedding (updated every N frames)
|
| 173 |
-
self._active_embeddings = {}
|
| 174 |
-
|
| 175 |
-
def load(self):
|
| 176 |
-
"""Load DINOv3 ViT-B/16 from HuggingFace."""
|
| 177 |
-
from transformers import AutoImageProcessor, AutoModel
|
| 178 |
-
|
| 179 |
-
# Path to the mounted container volume pointing to P:\Projects\Eagle Vision Project\models\dinov3-vitb16-pretrain-lvd1689m
|
| 180 |
-
model_id = "/models/dinov3-vitb16-pretrain-lvd1689m"
|
| 181 |
-
logger.info(f"Loading DINOv3 Re-ID model locally from: {model_id}")
|
| 182 |
-
|
| 183 |
-
self.processor = AutoImageProcessor.from_pretrained(model_id)
|
| 184 |
-
self.model = AutoModel.from_pretrained(model_id).to(self.device)
|
| 185 |
-
self.model.eval()
|
| 186 |
-
|
| 187 |
-
# Load contrastive-trained projection head if available
|
| 188 |
-
# Architecture must match training/train_reid.py
|
| 189 |
-
import torch.nn as nn
|
| 190 |
-
self.projection = nn.Sequential(
|
| 191 |
-
nn.Linear(1536, 768),
|
| 192 |
-
nn.BatchNorm1d(768),
|
| 193 |
-
nn.GELU(),
|
| 194 |
-
nn.Dropout(0.1),
|
| 195 |
-
nn.Linear(768, 256),
|
| 196 |
-
nn.BatchNorm1d(256),
|
| 197 |
-
nn.GELU(),
|
| 198 |
-
nn.Dropout(0.1),
|
| 199 |
-
nn.Linear(256, 128),
|
| 200 |
-
).to(self.device)
|
| 201 |
-
|
| 202 |
-
# Projection head disabled by default until trained with contrastive loss.
|
| 203 |
-
# To enable: set REID_USE_PROJECTION=1
|
| 204 |
-
use_projection = os.getenv('REID_USE_PROJECTION', '0') == '1'
|
| 205 |
-
if use_projection:
|
| 206 |
-
try:
|
| 207 |
-
proj_path = resolve_weights('dinov3_reid_head.pth', local_dir=os.getenv('MODEL_PATH', '/models'))
|
| 208 |
-
self.projection.load_state_dict(torch.load(proj_path, map_location=self.device))
|
| 209 |
-
self.projection.eval()
|
| 210 |
-
logger.info("Loaded contrastive Re-ID Projection Head (128-dim)")
|
| 211 |
-
except FileNotFoundError:
|
| 212 |
-
self.projection = None
|
| 213 |
-
logger.warning("Re-ID projection head not found — falling back to base DINOv3 1536-dim embeddings")
|
| 214 |
-
else:
|
| 215 |
-
self.projection = None
|
| 216 |
-
logger.info("Using base DINOv3 1536-dim embeddings (no projection head)")
|
| 217 |
-
|
| 218 |
-
logger.info(f"DINOv3 Re-ID backbone loaded on {self.device}")
|
| 219 |
-
|
| 220 |
-
def _crop_with_context(self, frame: np.ndarray, bbox: np.ndarray,
|
| 221 |
-
context_ratio: float = 0.15) -> np.ndarray:
|
| 222 |
-
"""
|
| 223 |
-
Crop equipment with context padding and square normalization.
|
| 224 |
-
Adding 15% context around the bbox captures surrounding details
|
| 225 |
-
(shadow pattern, ground type) that help stabilize embeddings.
|
| 226 |
-
"""
|
| 227 |
-
x1, y1, x2, y2 = bbox[:4].astype(int)
|
| 228 |
-
w, h = x2 - x1, y2 - y1
|
| 229 |
-
|
| 230 |
-
# Add context padding
|
| 231 |
-
pad_x = int(w * context_ratio)
|
| 232 |
-
pad_y = int(h * context_ratio)
|
| 233 |
-
x1 = max(0, x1 - pad_x)
|
| 234 |
-
y1 = max(0, y1 - pad_y)
|
| 235 |
-
x2 = min(frame.shape[1], x2 + pad_x)
|
| 236 |
-
y2 = min(frame.shape[0], y2 + pad_y)
|
| 237 |
-
|
| 238 |
-
crop = frame[y1:y2, x1:x2]
|
| 239 |
-
if crop.size == 0 or crop.shape[0] < 20 or crop.shape[1] < 20:
|
| 240 |
-
return None
|
| 241 |
-
|
| 242 |
-
# Pad to square to avoid aspect-ratio distortion
|
| 243 |
-
h, w = crop.shape[:2]
|
| 244 |
-
if h != w:
|
| 245 |
-
size = max(h, w)
|
| 246 |
-
padded = np.zeros((size, size, 3), dtype=crop.dtype)
|
| 247 |
-
y_off = (size - h) // 2
|
| 248 |
-
x_off = (size - w) // 2
|
| 249 |
-
padded[y_off:y_off + h, x_off:x_off + w] = crop
|
| 250 |
-
crop = padded
|
| 251 |
-
|
| 252 |
-
return crop
|
| 253 |
-
|
| 254 |
-
@torch.inference_mode()
|
| 255 |
-
def extract_embedding(self, frame: np.ndarray, bbox: np.ndarray) -> np.ndarray:
|
| 256 |
-
"""
|
| 257 |
-
Extract a 1536-d embedding from an equipment crop.
|
| 258 |
-
Uses CLS + mean(patch_tokens) with 15% context padding.
|
| 259 |
-
|
| 260 |
-
Args:
|
| 261 |
-
frame: Full BGR frame (H, W, 3)
|
| 262 |
-
bbox: [x1, y1, x2, y2]
|
| 263 |
-
|
| 264 |
-
Returns:
|
| 265 |
-
Normalized embedding vector, or None
|
| 266 |
-
"""
|
| 267 |
-
crop = self._crop_with_context(frame, bbox)
|
| 268 |
-
if crop is None:
|
| 269 |
-
return None
|
| 270 |
-
|
| 271 |
-
crop_rgb = cv2.cvtColor(crop, cv2.COLOR_BGR2RGB)
|
| 272 |
-
pil_img = Image.fromarray(crop_rgb)
|
| 273 |
-
inputs = self.processor(images=pil_img, return_tensors="pt").to(self.device)
|
| 274 |
-
outputs = self.model(**inputs)
|
| 275 |
-
|
| 276 |
-
# CLS + mean(patch_tokens) — DINOv3 ViT-B/16: 1 CLS + 4 register + 196 patch
|
| 277 |
-
hidden = outputs.last_hidden_state[0]
|
| 278 |
-
cls_token = hidden[0]
|
| 279 |
-
patch_tokens = hidden[5:]
|
| 280 |
-
mean_patches = patch_tokens.mean(dim=0)
|
| 281 |
-
embedding = torch.cat([cls_token, mean_patches])
|
| 282 |
-
|
| 283 |
-
if self.projection is not None:
|
| 284 |
-
import torch.nn.functional as F
|
| 285 |
-
features = embedding.unsqueeze(0).to(self.device)
|
| 286 |
-
projected = self.projection(features)
|
| 287 |
-
projected = F.normalize(projected, p=2, dim=1)
|
| 288 |
-
embedding = projected[0].cpu().numpy()
|
| 289 |
-
else:
|
| 290 |
-
embedding = embedding.cpu().numpy()
|
| 291 |
-
norm = np.linalg.norm(embedding)
|
| 292 |
-
if norm > 0:
|
| 293 |
-
embedding = embedding / norm
|
| 294 |
-
|
| 295 |
-
return embedding
|
| 296 |
-
|
| 297 |
-
def _extract_color_histogram(self, frame: np.ndarray, bbox: np.ndarray) -> np.ndarray:
|
| 298 |
-
"""
|
| 299 |
-
Extract an HSV color histogram from a crop. This is a hard physical
|
| 300 |
-
discriminator: a blue truck and a white truck will NEVER match,
|
| 301 |
-
regardless of what the neural network thinks.
|
| 302 |
-
"""
|
| 303 |
-
x1, y1, x2, y2 = bbox[:4].astype(int)
|
| 304 |
-
x1, y1 = max(0, x1), max(0, y1)
|
| 305 |
-
x2 = min(frame.shape[1], x2)
|
| 306 |
-
y2 = min(frame.shape[0], y2)
|
| 307 |
-
crop = frame[y1:y2, x1:x2]
|
| 308 |
-
if crop.size == 0:
|
| 309 |
-
return None
|
| 310 |
-
hsv = cv2.cvtColor(crop, cv2.COLOR_BGR2HSV)
|
| 311 |
-
# 2D histogram on Hue (color) and Saturation (vividness)
|
| 312 |
-
hist = cv2.calcHist([hsv], [0, 1], None, [30, 32], [0, 180, 0, 256])
|
| 313 |
-
cv2.normalize(hist, hist)
|
| 314 |
-
return hist.flatten()
|
| 315 |
-
|
| 316 |
-
def update_active(self, track_id: int, embedding: np.ndarray,
|
| 317 |
-
equipment_id: str, class_id: int, bbox: np.ndarray,
|
| 318 |
-
frame_id: int):
|
| 319 |
-
"""Store embedding for an active track. Keeps up to 5 diverse views."""
|
| 320 |
-
max_embeddings = 5
|
| 321 |
-
if track_id not in self._active_embeddings:
|
| 322 |
-
self._active_embeddings[track_id] = {
|
| 323 |
-
'embeddings': [embedding], # list of diverse views
|
| 324 |
-
'equipment_id': equipment_id,
|
| 325 |
-
'class_id': class_id,
|
| 326 |
-
'bbox': bbox.copy(),
|
| 327 |
-
'first_frame': frame_id,
|
| 328 |
-
'last_frame': frame_id,
|
| 329 |
-
'color_hist': None # populated by process_frame
|
| 330 |
-
}
|
| 331 |
-
else:
|
| 332 |
-
entry = self._active_embeddings[track_id]
|
| 333 |
-
entry['equipment_id'] = equipment_id
|
| 334 |
-
entry['bbox'] = bbox.copy()
|
| 335 |
-
entry['last_frame'] = frame_id
|
| 336 |
-
# Only add if sufficiently different from existing embeddings
|
| 337 |
-
existing = entry['embeddings']
|
| 338 |
-
max_sim = max(float(np.dot(embedding, e)) for e in existing)
|
| 339 |
-
if max_sim < 0.85 and len(existing) < max_embeddings:
|
| 340 |
-
existing.append(embedding) # New diverse view
|
| 341 |
-
elif len(existing) > 0:
|
| 342 |
-
existing[-1] = embedding # Refresh the most recent one
|
| 343 |
-
|
| 344 |
-
def move_to_gallery(self, track_id: int, frame_id: int):
|
| 345 |
-
"""
|
| 346 |
-
Move a lost track to the gallery for future re-identification.
|
| 347 |
-
Called when BoT-SORT drops a track.
|
| 348 |
-
Only moves tracks that have an equipment_id (confirmed tracks).
|
| 349 |
-
Pending tracks (no equipment_id yet) are silently dropped.
|
| 350 |
-
"""
|
| 351 |
-
if track_id in self._active_embeddings:
|
| 352 |
-
entry = self._active_embeddings.pop(track_id)
|
| 353 |
-
# Only gallery tracks that have a confirmed equipment ID
|
| 354 |
-
if entry.get('equipment_id') is None:
|
| 355 |
-
return # pending track — never got an ID, just drop it
|
| 356 |
-
entry['lost_frame'] = frame_id
|
| 357 |
-
self._gallery[track_id] = entry
|
| 358 |
-
logger.info(f"Track {track_id} ({entry['equipment_id']}) moved to Re-ID gallery (gallery size: {len(self._gallery)})")
|
| 359 |
-
|
| 360 |
-
def _compute_spatial_similarity(self, query_bbox, gallery_bbox, frame_width, frame_height):
|
| 361 |
-
"""
|
| 362 |
-
Compute spatial similarity between query and gallery bounding boxes.
|
| 363 |
-
Uses both center distance and IoU-like size similarity.
|
| 364 |
-
Returns 0.0 (far apart) to 1.0 (same position).
|
| 365 |
-
"""
|
| 366 |
-
if query_bbox is None or gallery_bbox is None:
|
| 367 |
-
return 0.0
|
| 368 |
-
|
| 369 |
-
# Center distance (normalized by frame diagonal)
|
| 370 |
-
lost_cx = (gallery_bbox[0] + gallery_bbox[2]) / 2
|
| 371 |
-
lost_cy = (gallery_bbox[1] + gallery_bbox[3]) / 2
|
| 372 |
-
new_cx = (query_bbox[0] + query_bbox[2]) / 2
|
| 373 |
-
new_cy = (query_bbox[1] + query_bbox[3]) / 2
|
| 374 |
-
|
| 375 |
-
diag = np.sqrt(frame_width**2 + frame_height**2)
|
| 376 |
-
dist = np.sqrt((new_cx - lost_cx)**2 + (new_cy - lost_cy)**2)
|
| 377 |
-
spatial_sim = max(0.0, 1.0 - dist / (diag * 0.5)) # 0 at half-diagonal away
|
| 378 |
-
|
| 379 |
-
return spatial_sim
|
| 380 |
-
|
| 381 |
-
def _get_alpha(self, frames_lost, fps=9.0):
|
| 382 |
-
"""
|
| 383 |
-
Time-decaying alpha for combining visual + spatial similarity.
|
| 384 |
-
For aerial drone footage, equipment barely moves during short track losses,
|
| 385 |
-
so spatial proximity is the dominant signal. Visual appearance changes
|
| 386 |
-
drastically with drone angle/lighting (same vehicle can score 0.40-0.80).
|
| 387 |
-
"""
|
| 388 |
-
seconds_lost = frames_lost / fps
|
| 389 |
-
if seconds_lost < 3.0:
|
| 390 |
-
return 0.3 # 30% visual, 70% spatial — equipment hasn't moved
|
| 391 |
-
elif seconds_lost < 10.0:
|
| 392 |
-
return 0.5 # Equal weight — vehicle may have moved somewhat
|
| 393 |
-
else:
|
| 394 |
-
return 0.8 # Mostly visual — vehicle could be anywhere
|
| 395 |
-
|
| 396 |
-
def query(self, embedding: np.ndarray, class_id: int,
|
| 397 |
-
frame_id: int, query_bbox: np.ndarray = None,
|
| 398 |
-
frame_width: int = 1920, frame_height: int = 1080,
|
| 399 |
-
query_color_hist: np.ndarray = None) -> tuple:
|
| 400 |
-
"""
|
| 401 |
-
Search the gallery for a matching lost track.
|
| 402 |
-
Combines visual embedding similarity with spatial proximity,
|
| 403 |
-
weighted by how long the track has been lost (time-decaying alpha).
|
| 404 |
-
|
| 405 |
-
final_score = α * visual_sim + (1-α) * spatial_sim
|
| 406 |
-
α increases with time lost (position becomes less reliable).
|
| 407 |
-
|
| 408 |
-
Args:
|
| 409 |
-
embedding: Query embedding from new detection
|
| 410 |
-
class_id: Class of the new detection
|
| 411 |
-
frame_id: Current frame number
|
| 412 |
-
query_bbox: [x1,y1,x2,y2] of the new detection
|
| 413 |
-
frame_width: Width of the video frame in pixels
|
| 414 |
-
frame_height: Height of the video frame in pixels
|
| 415 |
-
query_color_hist: HSV color histogram (unused, kept for API compat)
|
| 416 |
-
|
| 417 |
-
Returns:
|
| 418 |
-
(matched_equipment_id, final_score) or (None, 0.0)
|
| 419 |
-
"""
|
| 420 |
-
if embedding is None or len(self._gallery) == 0:
|
| 421 |
-
return None, 0.0
|
| 422 |
-
|
| 423 |
-
best_match_id = None
|
| 424 |
-
best_score = 0.0
|
| 425 |
-
second_best_score = 0.0
|
| 426 |
-
best_equip_id = None
|
| 427 |
-
best_visual = 0.0
|
| 428 |
-
best_spatial = 0.0
|
| 429 |
-
|
| 430 |
-
expired = []
|
| 431 |
-
for gal_track_id, entry in self._gallery.items():
|
| 432 |
-
# Check TTL
|
| 433 |
-
frames_lost = frame_id - entry.get('lost_frame', 0)
|
| 434 |
-
if frames_lost > self.gallery_ttl_frames:
|
| 435 |
-
expired.append(gal_track_id)
|
| 436 |
-
continue
|
| 437 |
-
|
| 438 |
-
# Only match same class
|
| 439 |
-
if entry['class_id'] != class_id:
|
| 440 |
-
continue
|
| 441 |
-
|
| 442 |
-
# ── Visual Similarity (best of stored embeddings) ──
|
| 443 |
-
visual_sim = max(
|
| 444 |
-
float(np.dot(embedding, e)) for e in entry['embeddings']
|
| 445 |
-
)
|
| 446 |
-
|
| 447 |
-
# ── Spatial Similarity ──
|
| 448 |
-
spatial_sim = self._compute_spatial_similarity(
|
| 449 |
-
query_bbox, entry.get('bbox'), frame_width, frame_height
|
| 450 |
-
)
|
| 451 |
-
|
| 452 |
-
# ── Combined Score (time-decaying alpha) ──
|
| 453 |
-
# BoT-SORT retains lost tracks for max_time_lost=1200 frames (20s at 60fps).
|
| 454 |
-
# Gallery entries are for tracks BoT-SORT has already dropped.
|
| 455 |
-
alpha = self._get_alpha(frames_lost)
|
| 456 |
-
final_score = alpha * visual_sim + (1 - alpha) * spatial_sim
|
| 457 |
-
|
| 458 |
-
# Hard floor: visual similarity must be at least 0.50
|
| 459 |
-
# Prevents position-dominant matches between different same-class vehicles
|
| 460 |
-
if visual_sim < 0.50:
|
| 461 |
-
continue
|
| 462 |
-
|
| 463 |
-
if final_score > best_score:
|
| 464 |
-
# Track second-best for margin-based disambiguation
|
| 465 |
-
second_best_score = best_score
|
| 466 |
-
best_score = final_score
|
| 467 |
-
best_match_id = gal_track_id
|
| 468 |
-
best_equip_id = entry['equipment_id']
|
| 469 |
-
best_visual = visual_sim
|
| 470 |
-
best_spatial = spatial_sim
|
| 471 |
-
elif final_score > second_best_score:
|
| 472 |
-
second_best_score = final_score
|
| 473 |
-
|
| 474 |
-
# Clean expired entries
|
| 475 |
-
for tid in expired:
|
| 476 |
-
del self._gallery[tid]
|
| 477 |
-
|
| 478 |
-
# ── Threshold decision ──
|
| 479 |
-
# BoT-SORT handles identity for up to 20s (max_time_lost=1200 at 60fps).
|
| 480 |
-
# Re-ID only fires for tracks lost longer than that — be strict to avoid
|
| 481 |
-
# false positives (different same-class vehicles getting the same ID).
|
| 482 |
-
if best_match_id is not None:
|
| 483 |
-
if best_score < self.similarity_threshold:
|
| 484 |
-
best_match_id = None # reject — below threshold
|
| 485 |
-
|
| 486 |
-
# Remove matched entry from gallery
|
| 487 |
-
if best_match_id is not None:
|
| 488 |
-
del self._gallery[best_match_id]
|
| 489 |
-
logger.info(
|
| 490 |
-
f"Re-ID match: {best_equip_id} "
|
| 491 |
-
f"(combined={best_score:.3f}, visual={best_visual:.3f}, spatial={best_spatial:.3f})"
|
| 492 |
-
)
|
| 493 |
-
else:
|
| 494 |
-
# Log why we didn't match
|
| 495 |
-
if len(self._gallery) > 0:
|
| 496 |
-
all_sims = []
|
| 497 |
-
for gal_entry in self._gallery.values():
|
| 498 |
-
if gal_entry['class_id'] == class_id:
|
| 499 |
-
v_sim = max(float(np.dot(embedding, emb)) for emb in gal_entry['embeddings'])
|
| 500 |
-
s_sim = self._compute_spatial_similarity(
|
| 501 |
-
query_bbox, gal_entry.get('bbox'), frame_width, frame_height
|
| 502 |
-
)
|
| 503 |
-
fl = frame_id - gal_entry.get('lost_frame', 0)
|
| 504 |
-
a = self._get_alpha(fl)
|
| 505 |
-
combined = a * v_sim + (1 - a) * s_sim
|
| 506 |
-
all_sims.append((gal_entry['equipment_id'], combined, v_sim, s_sim))
|
| 507 |
-
if all_sims:
|
| 508 |
-
best_below = max(all_sims, key=lambda x: x[1])
|
| 509 |
-
logger.info(
|
| 510 |
-
f"Re-ID no match: best={best_below[0]} "
|
| 511 |
-
f"combined={best_below[1]:.3f} (visual={best_below[2]:.3f}, spatial={best_below[3]:.3f}), "
|
| 512 |
-
f"threshold={self.similarity_threshold:.2f}"
|
| 513 |
-
)
|
| 514 |
-
|
| 515 |
-
return best_equip_id, best_score
|
| 516 |
-
|
| 517 |
-
def absorb_near_misses(self, track_id: int, embedding: np.ndarray,
|
| 518 |
-
class_id: int, frame_id: int,
|
| 519 |
-
query_bbox: np.ndarray = None,
|
| 520 |
-
frame_width: int = 1920, frame_height: int = 1080,
|
| 521 |
-
id_generator=None):
|
| 522 |
-
"""
|
| 523 |
-
When a new ID is minted (Re-ID missed), check if there are gallery entries
|
| 524 |
-
of the same class that scored close to the threshold. If a high-confidence
|
| 525 |
-
match is found (>0.85), retroactively reassign the new track to the old
|
| 526 |
-
equipment ID — fixing phantom IDs (WL-003 → WL-002).
|
| 527 |
-
For lower scores (>0.70), just absorb embeddings to help future matching.
|
| 528 |
-
"""
|
| 529 |
-
near_miss_floor = self.similarity_threshold - 0.05
|
| 530 |
-
candidates = []
|
| 531 |
-
|
| 532 |
-
for gal_track_id, entry in list(self._gallery.items()):
|
| 533 |
-
if entry['class_id'] != class_id:
|
| 534 |
-
continue
|
| 535 |
-
|
| 536 |
-
visual_sim = max(float(np.dot(embedding, e)) for e in entry['embeddings'])
|
| 537 |
-
if visual_sim < 0.35:
|
| 538 |
-
continue
|
| 539 |
-
|
| 540 |
-
spatial_sim = self._compute_spatial_similarity(
|
| 541 |
-
query_bbox, entry.get('bbox'), frame_width, frame_height
|
| 542 |
-
)
|
| 543 |
-
frames_lost = frame_id - entry.get('lost_frame', 0)
|
| 544 |
-
alpha = self._get_alpha(frames_lost)
|
| 545 |
-
combined = alpha * visual_sim + (1 - alpha) * spatial_sim
|
| 546 |
-
|
| 547 |
-
if combined >= near_miss_floor:
|
| 548 |
-
candidates.append((gal_track_id, entry, combined))
|
| 549 |
-
|
| 550 |
-
if not candidates:
|
| 551 |
-
return None
|
| 552 |
-
|
| 553 |
-
# Sort by score — best match first
|
| 554 |
-
candidates.sort(key=lambda x: x[2], reverse=True)
|
| 555 |
-
best_gal_id, best_entry, best_score = candidates[0]
|
| 556 |
-
|
| 557 |
-
# High-confidence near-miss: retroactively reassign ID
|
| 558 |
-
# This fixes phantom IDs (e.g., WL-003 → WL-002 when score=0.975)
|
| 559 |
-
reassigned_id = None
|
| 560 |
-
if best_score >= 0.85 and id_generator is not None:
|
| 561 |
-
old_equip_id = best_entry['equipment_id']
|
| 562 |
-
current_equip_id = id_generator.get_existing(track_id)
|
| 563 |
-
if current_equip_id and current_equip_id != old_equip_id:
|
| 564 |
-
id_generator.reassign(track_id, old_equip_id)
|
| 565 |
-
# Recycle the phantom ID number so next new vehicle reuses it
|
| 566 |
-
id_generator.recycle(current_equip_id)
|
| 567 |
-
# Update active embeddings with the corrected equipment ID
|
| 568 |
-
if track_id in self._active_embeddings:
|
| 569 |
-
self._active_embeddings[track_id]['equipment_id'] = old_equip_id
|
| 570 |
-
reassigned_id = old_equip_id
|
| 571 |
-
logger.info(
|
| 572 |
-
f"Re-ID retroactive fix: {current_equip_id} → {old_equip_id} "
|
| 573 |
-
f"(score={best_score:.3f}) — phantom ID corrected, "
|
| 574 |
-
f"{current_equip_id} recycled"
|
| 575 |
-
)
|
| 576 |
-
|
| 577 |
-
# Absorb embeddings from all near-misses
|
| 578 |
-
for gal_track_id, entry, combined in candidates:
|
| 579 |
-
if track_id in self._active_embeddings:
|
| 580 |
-
active = self._active_embeddings[track_id]
|
| 581 |
-
for old_emb in entry['embeddings']:
|
| 582 |
-
max_sim = max(float(np.dot(old_emb, e)) for e in active['embeddings'])
|
| 583 |
-
if max_sim < 0.90 and len(active['embeddings']) < 8:
|
| 584 |
-
active['embeddings'].append(old_emb)
|
| 585 |
-
del self._gallery[gal_track_id]
|
| 586 |
-
|
| 587 |
-
return reassigned_id
|
| 588 |
-
|
| 589 |
-
def cleanup(self, frame_id: int, id_generator=None):
|
| 590 |
-
"""Remove expired gallery entries and recycle their equipment IDs.
|
| 591 |
-
Self-healing: phantom IDs from false positives are recycled after TTL,
|
| 592 |
-
making their numbers available for real equipment."""
|
| 593 |
-
expired = [
|
| 594 |
-
tid for tid, entry in self._gallery.items()
|
| 595 |
-
if frame_id - entry['lost_frame'] > self.gallery_ttl_frames
|
| 596 |
-
]
|
| 597 |
-
for tid in expired:
|
| 598 |
-
entry = self._gallery[tid]
|
| 599 |
-
equip_id = entry.get('equipment_id')
|
| 600 |
-
if equip_id and id_generator:
|
| 601 |
-
id_generator.recycle(equip_id)
|
| 602 |
-
logger.info(f"Gallery TTL expired: {equip_id} recycled after {self.gallery_ttl_frames} frames")
|
| 603 |
-
del self._gallery[tid]
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
# =============================================================================
|
| 607 |
-
# TAI Heuristic (Track-Aware Initialization from TrackTrack, CVPR 2025)
|
| 608 |
-
# =============================================================================
|
| 609 |
-
def apply_tai_heuristic(detections: np.ndarray,
|
| 610 |
-
tracked_boxes: np.ndarray,
|
| 611 |
-
ioa_threshold: float = 0.4) -> np.ndarray:
|
| 612 |
-
"""
|
| 613 |
-
TAI port from TrackTrack (CVPR 2025).
|
| 614 |
-
Filters detections that are spatially contained inside existing tracks,
|
| 615 |
-
preventing fragmentary IDs (e.g., excavator bucket detected as separate equipment).
|
| 616 |
-
|
| 617 |
-
Args:
|
| 618 |
-
detections: (N, 6) array of [x1, y1, x2, y2, conf, class_id]
|
| 619 |
-
tracked_boxes: (M, 4) array of [x1, y1, x2, y2] from active tracks
|
| 620 |
-
ioa_threshold: Intersection-over-Area threshold (0.4 = 40% contained)
|
| 621 |
-
|
| 622 |
-
Returns:
|
| 623 |
-
Filtered detections (K, 6) where K <= N
|
| 624 |
-
"""
|
| 625 |
-
if len(tracked_boxes) == 0 or len(detections) == 0:
|
| 626 |
-
return detections
|
| 627 |
-
|
| 628 |
-
filtered = []
|
| 629 |
-
for det in detections:
|
| 630 |
-
det_box = det[:4]
|
| 631 |
-
|
| 632 |
-
# Calculate IoA: intersection / area_of_detection
|
| 633 |
-
# (How much of the detection is inside an existing track?)
|
| 634 |
-
det_area = max((det_box[2] - det_box[0]) * (det_box[3] - det_box[1]), 1e-6)
|
| 635 |
-
|
| 636 |
-
is_fragment = False
|
| 637 |
-
for track_box in tracked_boxes:
|
| 638 |
-
track_area = max(
|
| 639 |
-
(track_box[2] - track_box[0]) * (track_box[3] - track_box[1]), 1e-6
|
| 640 |
-
)
|
| 641 |
-
|
| 642 |
-
# Only consider fragment filtering if the detection is
|
| 643 |
-
# significantly SMALLER than the track (< 50% of track area).
|
| 644 |
-
# Full-size re-detections of the same equipment should pass through.
|
| 645 |
-
if det_area > 0.5 * track_area:
|
| 646 |
-
continue
|
| 647 |
-
|
| 648 |
-
# Intersection
|
| 649 |
-
x1 = max(det_box[0], track_box[0])
|
| 650 |
-
y1 = max(det_box[1], track_box[1])
|
| 651 |
-
x2 = min(det_box[2], track_box[2])
|
| 652 |
-
y2 = min(det_box[3], track_box[3])
|
| 653 |
-
|
| 654 |
-
inter_area = max(0, x2 - x1) * max(0, y2 - y1)
|
| 655 |
-
ioa = inter_area / det_area
|
| 656 |
-
|
| 657 |
-
if ioa > ioa_threshold:
|
| 658 |
-
is_fragment = True
|
| 659 |
-
break
|
| 660 |
-
|
| 661 |
-
if not is_fragment:
|
| 662 |
-
filtered.append(det)
|
| 663 |
-
|
| 664 |
-
if len(filtered) == 0:
|
| 665 |
-
return np.empty((0, 6))
|
| 666 |
-
return np.array(filtered)
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
# =============================================================================
|
| 670 |
-
# Grid-Based Articulated Motion Detection (Farneback Optical Flow)
|
| 671 |
-
# =============================================================================
|
| 672 |
-
class ArticulatedMotionDetector:
|
| 673 |
-
"""
|
| 674 |
-
Detects motion within equipment bounding boxes using
|
| 675 |
-
Grid Decomposition (3x3) + Farneback Dense Optical Flow.
|
| 676 |
-
|
| 677 |
-
Equipment-agnostic: works for excavators (arm-only motion),
|
| 678 |
-
wheel loaders (bucket pivot), dump trucks (driving), etc.
|
| 679 |
-
No assumptions about which grid region corresponds to which part.
|
| 680 |
-
"""
|
| 681 |
-
|
| 682 |
-
def __init__(self, grid_size: int = 3, motion_threshold: float = 0.15):
|
| 683 |
-
self.grid_size = grid_size
|
| 684 |
-
self.motion_threshold = motion_threshold
|
| 685 |
-
self._prev_crops = {} # track_id -> previous grayscale crop
|
| 686 |
-
self._centroid_history = {} # track_id -> deque of (cx, cy)
|
| 687 |
-
self._centroid_window = 30 # 0.5s at 60fps
|
| 688 |
-
self._active_history = {} # track_id -> deque of recent is_active bools
|
| 689 |
-
self._active_smoothing = 60 # 1 second at 60fps — bridges intra-cycle pauses
|
| 690 |
-
|
| 691 |
-
def detect(self, frame_gray: np.ndarray, track_id: int,
|
| 692 |
-
bbox: np.ndarray) -> dict:
|
| 693 |
-
"""
|
| 694 |
-
Analyze motion within a bounding box.
|
| 695 |
-
|
| 696 |
-
Returns:
|
| 697 |
-
{
|
| 698 |
-
'is_active': bool,
|
| 699 |
-
'motion_source': str ('partial', 'whole_body', 'none'),
|
| 700 |
-
'cell_motions': list[list[float]], # 3x3 magnitude grid
|
| 701 |
-
'cell_directions': list[list[list[float]]], # 3x3x2 flow direction grid
|
| 702 |
-
'max_flow': float,
|
| 703 |
-
'active_cell_ratio': float # fraction of cells with motion
|
| 704 |
-
}
|
| 705 |
-
"""
|
| 706 |
-
x1, y1, x2, y2 = bbox[:4].astype(int)
|
| 707 |
-
x1, y1 = max(0, x1), max(0, y1)
|
| 708 |
-
x2 = min(frame_gray.shape[1], x2)
|
| 709 |
-
y2 = min(frame_gray.shape[0], y2)
|
| 710 |
-
|
| 711 |
-
# --- Centroid displacement over rolling window ---
|
| 712 |
-
cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
|
| 713 |
-
centroid_moving = False
|
| 714 |
-
if track_id not in self._centroid_history:
|
| 715 |
-
self._centroid_history[track_id] = deque(maxlen=self._centroid_window + 1)
|
| 716 |
-
self._centroid_history[track_id].append((cx, cy))
|
| 717 |
-
|
| 718 |
-
history = self._centroid_history[track_id]
|
| 719 |
-
if len(history) >= 2:
|
| 720 |
-
old_cx, old_cy = history[0]
|
| 721 |
-
displacement = np.sqrt((cx - old_cx) ** 2 + (cy - old_cy) ** 2)
|
| 722 |
-
if displacement > 15.0:
|
| 723 |
-
centroid_moving = True
|
| 724 |
-
|
| 725 |
-
crop = frame_gray[y1:y2, x1:x2]
|
| 726 |
-
|
| 727 |
-
if crop.size == 0 or crop.shape[0] < 10 or crop.shape[1] < 10:
|
| 728 |
-
return self._empty_result(centroid_moving)
|
| 729 |
-
|
| 730 |
-
# Resize crop to standard size — this inherently normalizes for scale.
|
| 731 |
-
# Large boxes (close equipment) and small boxes (distant) produce
|
| 732 |
-
# comparable flow magnitudes after resize. No additional normalization needed.
|
| 733 |
-
crop_resized = self._resize_crop(crop, 300)
|
| 734 |
-
|
| 735 |
-
if track_id not in self._prev_crops:
|
| 736 |
-
self._prev_crops[track_id] = crop_resized
|
| 737 |
-
return self._empty_result(centroid_moving)
|
| 738 |
-
|
| 739 |
-
prev_crop = self._prev_crops[track_id]
|
| 740 |
-
self._prev_crops[track_id] = crop_resized
|
| 741 |
-
|
| 742 |
-
if prev_crop.shape != crop_resized.shape:
|
| 743 |
-
return self._empty_result(centroid_moving)
|
| 744 |
-
|
| 745 |
-
# Farneback dense optical flow
|
| 746 |
-
flow = cv2.calcOpticalFlowFarneback(
|
| 747 |
-
prev_crop, crop_resized,
|
| 748 |
-
None,
|
| 749 |
-
pyr_scale=0.5, levels=3, winsize=15,
|
| 750 |
-
iterations=3, poly_n=5, poly_sigma=1.2,
|
| 751 |
-
flags=0
|
| 752 |
-
)
|
| 753 |
-
|
| 754 |
-
# ── Global motion detection (ACTIVE/INACTIVE) ──
|
| 755 |
-
# Dual-gate approach: detect motion via EITHER of two independent signals.
|
| 756 |
-
# This prevents the failure mode where one metric collapses while
|
| 757 |
-
# the equipment is clearly active.
|
| 758 |
-
all_magnitudes = np.sqrt(flow[..., 0] ** 2 + flow[..., 1] ** 2)
|
| 759 |
-
top_flow = float(np.percentile(all_magnitudes, 95))
|
| 760 |
-
median_flow = float(np.median(all_magnitudes))
|
| 761 |
-
localized_motion = top_flow - median_flow
|
| 762 |
-
|
| 763 |
-
# Gate 1: Localized motion (top5% - median > threshold)
|
| 764 |
-
# Catches articulated motion (arm/bucket) when background is stable.
|
| 765 |
-
is_localized = localized_motion > self.motion_threshold
|
| 766 |
-
|
| 767 |
-
# Gate 2: Absolute flow magnitude (top5% > 0.35)
|
| 768 |
-
# Catches ALL significant motion regardless of median elevation.
|
| 769 |
-
# Parked equipment: top5% ≈ 0.14-0.29 (max observed 0.29).
|
| 770 |
-
# Bucket-only tip-in-place: top5% ≈ 0.35-0.60.
|
| 771 |
-
# Active equipment: top5% ≈ 0.5-1.2.
|
| 772 |
-
# 0.35 gives 0.06 margin above highest parked value while catching
|
| 773 |
-
# the tip-in-place phase that was draining the counter.
|
| 774 |
-
is_strong_flow = top_flow > 0.35
|
| 775 |
-
|
| 776 |
-
is_active_flow = is_localized or is_strong_flow
|
| 777 |
-
|
| 778 |
-
# ── Grid decomposition (for activity classification) ──
|
| 779 |
-
h, w = crop_resized.shape[:2]
|
| 780 |
-
cell_h, cell_w = h // self.grid_size, w // self.grid_size
|
| 781 |
-
|
| 782 |
-
cell_motions = []
|
| 783 |
-
cell_directions = []
|
| 784 |
-
|
| 785 |
-
for row in range(self.grid_size):
|
| 786 |
-
motion_row = []
|
| 787 |
-
direction_row = []
|
| 788 |
-
for col in range(self.grid_size):
|
| 789 |
-
cy1 = row * cell_h
|
| 790 |
-
cy2 = (row + 1) * cell_h if row < self.grid_size - 1 else h
|
| 791 |
-
cx1 = col * cell_w
|
| 792 |
-
cx2 = (col + 1) * cell_w if col < self.grid_size - 1 else w
|
| 793 |
-
|
| 794 |
-
cell_flow = flow[cy1:cy2, cx1:cx2]
|
| 795 |
-
magnitudes = np.sqrt(
|
| 796 |
-
cell_flow[..., 0] ** 2 + cell_flow[..., 1] ** 2
|
| 797 |
-
)
|
| 798 |
-
|
| 799 |
-
p95_mag = float(np.percentile(magnitudes, 95))
|
| 800 |
-
motion_row.append(p95_mag)
|
| 801 |
-
|
| 802 |
-
avg_dx = float(np.mean(cell_flow[..., 0]))
|
| 803 |
-
avg_dy = float(np.mean(cell_flow[..., 1]))
|
| 804 |
-
direction_row.append([avg_dx, avg_dy])
|
| 805 |
-
|
| 806 |
-
cell_motions.append(motion_row)
|
| 807 |
-
cell_directions.append(direction_row)
|
| 808 |
-
|
| 809 |
-
flat = np.array(cell_motions)
|
| 810 |
-
active_cells = flat > self.motion_threshold
|
| 811 |
-
num_active = int(np.sum(active_cells))
|
| 812 |
-
total_cells = self.grid_size * self.grid_size
|
| 813 |
-
active_ratio = num_active / total_cells
|
| 814 |
-
|
| 815 |
-
raw_active = is_active_flow or centroid_moving
|
| 816 |
-
|
| 817 |
-
# ── Asymmetric hysteresis ──
|
| 818 |
-
# Construction equipment works in cycles (scoop-lift-dump-return).
|
| 819 |
-
# Brief pauses between phases should NOT flip state to INACTIVE.
|
| 820 |
-
# INACTIVE → ACTIVE: need 3/15 recent frames active (20%) — respond quickly
|
| 821 |
-
# ACTIVE → INACTIVE: need 12/15 recent frames inactive (80%) — hold through pauses
|
| 822 |
-
if track_id not in self._active_history:
|
| 823 |
-
self._active_history[track_id] = deque(maxlen=self._active_smoothing)
|
| 824 |
-
self._active_history[track_id].append(raw_active)
|
| 825 |
-
active_votes = sum(self._active_history[track_id])
|
| 826 |
-
total_votes = len(self._active_history[track_id])
|
| 827 |
-
|
| 828 |
-
# Asymmetric thresholds based on current state
|
| 829 |
-
if not hasattr(self, '_prev_smoothed'):
|
| 830 |
-
self._prev_smoothed = {}
|
| 831 |
-
prev_active = self._prev_smoothed.get(track_id, False)
|
| 832 |
-
if prev_active:
|
| 833 |
-
# Currently ACTIVE — slow decay (stay active if ≥8/60 recent frames active)
|
| 834 |
-
is_active = active_votes >= 8
|
| 835 |
-
else:
|
| 836 |
-
# Currently INACTIVE — fast attack (activate if ≥3/15 recent frames active)
|
| 837 |
-
recent_votes = sum(list(self._active_history[track_id])[-15:])
|
| 838 |
-
is_active = recent_votes >= 3
|
| 839 |
-
self._prev_smoothed[track_id] = is_active
|
| 840 |
-
|
| 841 |
-
# Motion source classification
|
| 842 |
-
if centroid_moving or active_ratio > 0.5:
|
| 843 |
-
motion_source = 'whole_body'
|
| 844 |
-
elif is_active:
|
| 845 |
-
motion_source = 'partial'
|
| 846 |
-
else:
|
| 847 |
-
motion_source = 'none'
|
| 848 |
-
|
| 849 |
-
# Debug logging
|
| 850 |
-
if not hasattr(self, '_debug_counter'):
|
| 851 |
-
self._debug_counter = 0
|
| 852 |
-
self._debug_counter += 1
|
| 853 |
-
if self._debug_counter % 30 == 0:
|
| 854 |
-
gate = "LOC" if is_localized else ("ABS" if is_strong_flow else ("CEN" if centroid_moving else "---"))
|
| 855 |
-
logger.info(
|
| 856 |
-
f"Motion[T{track_id}]: top5%={top_flow:.3f}, localized={localized_motion:.3f}, "
|
| 857 |
-
f"gate={gate}, raw={raw_active}, smoothed={is_active} ({active_votes}/{total_votes}), "
|
| 858 |
-
f"crop={crop_resized.shape}"
|
| 859 |
-
)
|
| 860 |
-
|
| 861 |
-
return {
|
| 862 |
-
'is_active': is_active,
|
| 863 |
-
'motion_source': motion_source,
|
| 864 |
-
'cell_motions': cell_motions,
|
| 865 |
-
'cell_directions': cell_directions,
|
| 866 |
-
'max_flow': top_flow,
|
| 867 |
-
'localized_motion': localized_motion,
|
| 868 |
-
'active_cell_ratio': active_ratio
|
| 869 |
-
}
|
| 870 |
-
|
| 871 |
-
def remove_track(self, track_id: int):
|
| 872 |
-
self._prev_crops.pop(track_id, None)
|
| 873 |
-
self._centroid_history.pop(track_id, None)
|
| 874 |
-
self._active_history.pop(track_id, None)
|
| 875 |
-
if hasattr(self, '_prev_smoothed'):
|
| 876 |
-
self._prev_smoothed.pop(track_id, None)
|
| 877 |
-
|
| 878 |
-
def _resize_crop(self, crop: np.ndarray, target_size: int) -> np.ndarray:
|
| 879 |
-
return cv2.resize(crop, (target_size, target_size), interpolation=cv2.INTER_LINEAR)
|
| 880 |
-
|
| 881 |
-
def _empty_result(self, centroid_moving: bool = False) -> dict:
|
| 882 |
-
empty_grid = [[0.0] * self.grid_size for _ in range(self.grid_size)]
|
| 883 |
-
empty_dirs = [[[0.0, 0.0]] * self.grid_size for _ in range(self.grid_size)]
|
| 884 |
-
return {
|
| 885 |
-
'is_active': centroid_moving,
|
| 886 |
-
'motion_source': 'whole_body' if centroid_moving else 'none',
|
| 887 |
-
'cell_motions': empty_grid,
|
| 888 |
-
'cell_directions': empty_dirs,
|
| 889 |
-
'max_flow': 0.0,
|
| 890 |
-
'localized_motion': 0.0,
|
| 891 |
-
'active_cell_ratio': 0.0
|
| 892 |
-
}
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
# =============================================================================
|
| 896 |
-
# Rule-Based Activity Classifier
|
| 897 |
-
# =============================================================================
|
| 898 |
-
class RuleBasedActivityClassifier:
|
| 899 |
-
"""
|
| 900 |
-
Per-track classifier: only determines ACTIVE/INACTIVE → WAITING.
|
| 901 |
-
|
| 902 |
-
Activity sub-labels (DIGGING, LOADING, DUMPING) are assigned in Phase 10
|
| 903 |
-
using spatial context — specifically, the equipment's proximity to dump
|
| 904 |
-
trucks. The flow ratio cannot distinguish DIGGING from DUMPING because
|
| 905 |
-
both produce identical signal topology (concentrated bucket motion).
|
| 906 |
-
Spatial context resolves this: at the pile = DIGGING, at the truck = DUMPING.
|
| 907 |
-
"""
|
| 908 |
-
|
| 909 |
-
def __init__(self):
|
| 910 |
-
pass
|
| 911 |
-
|
| 912 |
-
def classify_smoothed(self, track_id: int, motion_result: dict,
|
| 913 |
-
equipment_class: str = '') -> str:
|
| 914 |
-
"""Return WAITING if inactive, ACTIVE placeholder otherwise.
|
| 915 |
-
Phase 10 will assign the real activity sub-label."""
|
| 916 |
-
if not motion_result['is_active']:
|
| 917 |
-
return 'WAITING'
|
| 918 |
-
# Placeholder — Phase 10 spatial context replaces this
|
| 919 |
-
return 'LOADING'
|
| 920 |
-
|
| 921 |
-
def remove_track(self, track_id: int):
|
| 922 |
-
pass
|
| 923 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 924 |
|
| 925 |
-
|
| 926 |
-
# Video Activity Classifier (Option B — X3D-S)
|
| 927 |
-
# =============================================================================
|
| 928 |
-
class VideoActivityClassifier:
|
| 929 |
-
"""
|
| 930 |
-
Classifies equipment activity from 16-frame crop buffers using a
|
| 931 |
-
fine-tuned X3D-S model (Kinetics-400 pretrained → 4 activity classes).
|
| 932 |
|
| 933 |
-
|
| 934 |
-
reaches `clip_length` frames, runs inference and returns the predicted
|
| 935 |
-
activity label. Falls back to rule-based Phase 10 if model is not loaded.
|
| 936 |
|
| 937 |
-
|
| 938 |
-
"""
|
| 939 |
|
| 940 |
-
|
| 941 |
|
| 942 |
-
|
| 943 |
-
|
| 944 |
-
|
| 945 |
-
|
| 946 |
-
|
| 947 |
-
|
| 948 |
-
self.confidence_threshold = confidence_threshold
|
| 949 |
-
self.device = device if torch.cuda.is_available() else 'cpu'
|
| 950 |
-
self.model = None
|
| 951 |
-
self.weights_path = weights_path
|
| 952 |
|
| 953 |
-
|
| 954 |
-
self._buffers = {}
|
| 955 |
-
# Per-track last prediction: track_id -> (activity, confidence)
|
| 956 |
-
self._last_pred = {}
|
| 957 |
|
| 958 |
-
|
| 959 |
-
self._mean = torch.tensor([0.45, 0.45, 0.45]).view(1, 3, 1, 1)
|
| 960 |
-
self._std = torch.tensor([0.225, 0.225, 0.225]).view(1, 3, 1, 1)
|
| 961 |
|
| 962 |
-
|
| 963 |
-
"""Load the fine-tuned X3D-S model."""
|
| 964 |
-
if not self.weights_path or not os.path.exists(self.weights_path):
|
| 965 |
-
logger.warning(f"Activity classifier weights not found: {self.weights_path}")
|
| 966 |
-
logger.warning("Falling back to rule-based Phase 10 classification")
|
| 967 |
-
return False
|
| 968 |
|
| 969 |
-
|
| 970 |
-
|
| 971 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 972 |
|
| 973 |
-
|
| 974 |
|
| 975 |
-
|
| 976 |
-
|
| 977 |
-
|
| 978 |
-
|
| 979 |
-
|
| 980 |
-
|
| 981 |
|
| 982 |
-
|
| 983 |
-
self._mean = self._mean.to(self.device)
|
| 984 |
-
self._std = self._std.to(self.device)
|
| 985 |
|
| 986 |
-
|
| 987 |
-
logger.info(f"Activity classifier loaded: X3D-S, val_acc={val_acc}%")
|
| 988 |
-
return True
|
| 989 |
-
except Exception as e:
|
| 990 |
-
logger.error(f"Failed to load activity classifier: {e}")
|
| 991 |
-
self.model = None
|
| 992 |
-
return False
|
| 993 |
|
| 994 |
-
|
| 995 |
-
def is_loaded(self):
|
| 996 |
-
return self.model is not None
|
| 997 |
|
| 998 |
-
|
| 999 |
-
|
| 1000 |
-
|
| 1001 |
-
|
| 1002 |
-
track_id: Tracker-assigned ID
|
| 1003 |
-
crop: BGR crop from cv2, any size (will be resized)
|
| 1004 |
-
"""
|
| 1005 |
-
if crop.size == 0 or crop.shape[0] < 10 or crop.shape[1] < 10:
|
| 1006 |
-
return
|
| 1007 |
|
| 1008 |
-
|
| 1009 |
-
resized = cv2.resize(crop, (self.crop_size, self.crop_size))
|
| 1010 |
-
# BGR → RGB
|
| 1011 |
-
resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
|
| 1012 |
|
| 1013 |
-
|
| 1014 |
-
|
| 1015 |
-
self._buffers[track_id].append(resized)
|
| 1016 |
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
-
|
|
|
|
|
|
|
| 1020 |
|
| 1021 |
-
|
| 1022 |
-
(activity_label, confidence) or (None, 0.0) if not enough frames.
|
| 1023 |
-
activity_label is one of: DIGGING, LOADING, DUMPING, WAITING
|
| 1024 |
-
"""
|
| 1025 |
-
if not self.is_loaded:
|
| 1026 |
-
return None, 0.0
|
| 1027 |
|
| 1028 |
-
|
| 1029 |
-
if len(buf) < self.clip_length:
|
| 1030 |
-
# Not enough frames yet — return last prediction if available
|
| 1031 |
-
return self._last_pred.get(track_id, (None, 0.0))
|
| 1032 |
|
| 1033 |
-
|
| 1034 |
-
|
|
|
|
|
|
|
| 1035 |
|
| 1036 |
-
|
| 1037 |
-
frames_np = np.stack(clip_frames, axis=0) # (T, H, W, 3)
|
| 1038 |
-
clip_tensor = torch.from_numpy(frames_np).float() / 255.0 # (T, H, W, C)
|
| 1039 |
-
clip_tensor = clip_tensor.permute(3, 0, 1, 2) # (C, T, H, W)
|
| 1040 |
|
| 1041 |
-
|
| 1042 |
-
clip_tensor = (clip_tensor - self._mean.cpu()) / self._std.cpu()
|
| 1043 |
-
clip_tensor = clip_tensor.unsqueeze(0).to(self.device) # (1, C, T, H, W)
|
| 1044 |
|
| 1045 |
-
|
| 1046 |
-
|
| 1047 |
-
|
| 1048 |
-
|
| 1049 |
-
pred_idx = probs.argmax(1).item()
|
| 1050 |
-
confidence = probs[0, pred_idx].item()
|
| 1051 |
|
| 1052 |
-
|
|
|
|
| 1053 |
|
| 1054 |
-
|
| 1055 |
-
|
| 1056 |
-
# Low confidence — keep previous prediction or default
|
| 1057 |
-
prev = self._last_pred.get(track_id, (None, 0.0))
|
| 1058 |
-
if prev[0] is not None:
|
| 1059 |
-
activity, confidence = prev
|
| 1060 |
|
| 1061 |
-
|
|
|
|
|
|
|
|
|
|
| 1062 |
|
| 1063 |
-
|
| 1064 |
-
self._buffers[track_id] = buf[self.clip_stride:]
|
| 1065 |
|
| 1066 |
-
|
| 1067 |
|
| 1068 |
-
|
| 1069 |
-
"""Clean up state for a lost track."""
|
| 1070 |
-
self._buffers.pop(track_id, None)
|
| 1071 |
-
self._last_pred.pop(track_id, None)
|
| 1072 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1073 |
|
| 1074 |
-
|
| 1075 |
-
# Equipment ID Generator
|
| 1076 |
-
# =============================================================================
|
| 1077 |
-
class EquipmentIDGenerator:
|
| 1078 |
-
"""Generates class-prefixed equipment IDs: EX-001, DT-001, BD-001 (per-class counters)."""
|
| 1079 |
-
|
| 1080 |
-
# Minimum confidence to hard-lock class at creation. Below this,
|
| 1081 |
-
# class can be corrected by a majority vote over the first N frames.
|
| 1082 |
-
CLASS_LOCK_CONF_THRESHOLD = 0.55
|
| 1083 |
-
|
| 1084 |
-
def __init__(self):
|
| 1085 |
-
# Audit #7: per-class counters instead of global — avoids inflated IDs
|
| 1086 |
-
self._counters = {} # prefix -> int
|
| 1087 |
-
self._track_to_equip = {} # tracker_id -> equipment_id
|
| 1088 |
-
self._equip_to_class = {} # equipment_id -> class_id (locked at creation)
|
| 1089 |
-
self._class_locked = {} # equipment_id -> bool (True = hard-locked)
|
| 1090 |
-
self._class_votes = {} # equipment_id -> list of (class_id, conf) for soft-lock
|
| 1091 |
-
self._recycled = {} # prefix -> sorted list of recycled numbers
|
| 1092 |
-
|
| 1093 |
-
def get_existing(self, track_id: int) -> str | None:
|
| 1094 |
-
"""Get existing equipment ID for this track, or None if new."""
|
| 1095 |
-
return self._track_to_equip.get(track_id)
|
| 1096 |
-
|
| 1097 |
-
def get_or_create(self, track_id: int, class_id: int, conf: float = 1.0) -> str:
|
| 1098 |
-
"""Get existing equipment ID or create a new one for this track."""
|
| 1099 |
-
if track_id in self._track_to_equip:
|
| 1100 |
-
return self._track_to_equip[track_id]
|
| 1101 |
-
|
| 1102 |
-
prefix = CLASS_PREFIXES.get(int(class_id), 'UK')
|
| 1103 |
-
self._counters.setdefault(prefix, 0)
|
| 1104 |
-
|
| 1105 |
-
# Use recycled number if available, otherwise increment counter
|
| 1106 |
-
if prefix in self._recycled and self._recycled[prefix]:
|
| 1107 |
-
num = self._recycled[prefix].pop(0)
|
| 1108 |
-
else:
|
| 1109 |
-
self._counters[prefix] += 1
|
| 1110 |
-
num = self._counters[prefix]
|
| 1111 |
-
|
| 1112 |
-
equip_id = f"{prefix}-{num:03d}"
|
| 1113 |
-
self._track_to_equip[track_id] = equip_id
|
| 1114 |
-
self._equip_to_class[equip_id] = class_id
|
| 1115 |
-
|
| 1116 |
-
# Hard-lock class only if initial detection confidence is high enough
|
| 1117 |
-
if conf >= self.CLASS_LOCK_CONF_THRESHOLD:
|
| 1118 |
-
self._class_locked[equip_id] = True
|
| 1119 |
-
else:
|
| 1120 |
-
self._class_locked[equip_id] = False
|
| 1121 |
-
self._class_votes[equip_id] = [(class_id, conf)]
|
| 1122 |
-
logger.info(f"{equip_id} class soft-locked (conf={conf:.2f} < {self.CLASS_LOCK_CONF_THRESHOLD}), may be corrected")
|
| 1123 |
-
|
| 1124 |
-
return equip_id
|
| 1125 |
-
|
| 1126 |
-
def reassign(self, new_track_id: int, equip_id: str):
|
| 1127 |
-
"""Reassign an equipment ID to a new track (after Re-ID match)."""
|
| 1128 |
-
self._track_to_equip[new_track_id] = equip_id
|
| 1129 |
-
|
| 1130 |
-
def recycle(self, equip_id: str):
|
| 1131 |
-
"""Recycle a phantom equipment ID so its number can be reused.
|
| 1132 |
-
Called when absorb_near_misses reassigns a phantom to an older ID."""
|
| 1133 |
-
parts = equip_id.split('-')
|
| 1134 |
-
if len(parts) == 2:
|
| 1135 |
-
prefix = parts[0]
|
| 1136 |
-
num = int(parts[1])
|
| 1137 |
-
self._recycled.setdefault(prefix, [])
|
| 1138 |
-
if num not in self._recycled[prefix]:
|
| 1139 |
-
self._recycled[prefix].append(num)
|
| 1140 |
-
self._recycled[prefix].sort()
|
| 1141 |
-
# Clean up the phantom from equip_to_class
|
| 1142 |
-
self._equip_to_class.pop(equip_id, None)
|
| 1143 |
-
logger.info(f"Recycled phantom ID {equip_id} — number available for reuse")
|
| 1144 |
-
|
| 1145 |
-
def get_locked_class(self, equip_id: str) -> int | None:
|
| 1146 |
-
"""Get the class_id that was locked when this equipment ID was created."""
|
| 1147 |
-
return self._equip_to_class.get(equip_id)
|
| 1148 |
-
|
| 1149 |
-
def is_class_locked(self, equip_id: str) -> bool:
|
| 1150 |
-
"""True if this equipment's class is hard-locked (high initial confidence)."""
|
| 1151 |
-
return self._class_locked.get(equip_id, True)
|
| 1152 |
-
|
| 1153 |
-
def vote_class(self, track_id: int, equip_id: str, class_id: int, conf: float):
|
| 1154 |
-
"""For soft-locked equipment, accumulate class votes over first 10 detections.
|
| 1155 |
-
After 10 votes, pick the weighted majority class and hard-lock it.
|
| 1156 |
-
If the majority differs from the initial class, reassign the equipment ID."""
|
| 1157 |
-
if self._class_locked.get(equip_id, True):
|
| 1158 |
-
return # already hard-locked
|
| 1159 |
-
|
| 1160 |
-
votes = self._class_votes.get(equip_id, [])
|
| 1161 |
-
votes.append((class_id, conf))
|
| 1162 |
-
self._class_votes[equip_id] = votes
|
| 1163 |
-
|
| 1164 |
-
# After 10 observations, hard-lock to weighted majority class
|
| 1165 |
-
if len(votes) >= 10:
|
| 1166 |
-
# Weighted vote: sum confidence per class
|
| 1167 |
-
class_weights = {}
|
| 1168 |
-
for cid, c in votes:
|
| 1169 |
-
class_weights[cid] = class_weights.get(cid, 0.0) + c
|
| 1170 |
-
winner = max(class_weights, key=class_weights.get)
|
| 1171 |
-
current_class = self._equip_to_class.get(equip_id)
|
| 1172 |
-
|
| 1173 |
-
if winner != current_class:
|
| 1174 |
-
# Class correction needed — change the equipment ID prefix
|
| 1175 |
-
old_prefix = CLASS_PREFIXES.get(int(current_class), 'UK')
|
| 1176 |
-
new_prefix = CLASS_PREFIXES.get(int(winner), 'UK')
|
| 1177 |
-
|
| 1178 |
-
# Recycle the old number
|
| 1179 |
-
old_parts = equip_id.split('-')
|
| 1180 |
-
old_num = int(old_parts[1])
|
| 1181 |
-
self._recycled.setdefault(old_prefix, [])
|
| 1182 |
-
if old_num not in self._recycled[old_prefix]:
|
| 1183 |
-
self._recycled[old_prefix].append(old_num)
|
| 1184 |
-
self._recycled[old_prefix].sort()
|
| 1185 |
-
|
| 1186 |
-
# Allocate new number with correct prefix
|
| 1187 |
-
self._counters.setdefault(new_prefix, 0)
|
| 1188 |
-
if new_prefix in self._recycled and self._recycled[new_prefix]:
|
| 1189 |
-
new_num = self._recycled[new_prefix].pop(0)
|
| 1190 |
-
else:
|
| 1191 |
-
self._counters[new_prefix] += 1
|
| 1192 |
-
new_num = self._counters[new_prefix]
|
| 1193 |
-
|
| 1194 |
-
new_equip_id = f"{new_prefix}-{new_num:03d}"
|
| 1195 |
-
self._track_to_equip[track_id] = new_equip_id
|
| 1196 |
-
self._equip_to_class[new_equip_id] = winner
|
| 1197 |
-
self._class_locked[new_equip_id] = True
|
| 1198 |
-
# Clean up old
|
| 1199 |
-
self._equip_to_class.pop(equip_id, None)
|
| 1200 |
-
self._class_locked.pop(equip_id, None)
|
| 1201 |
-
self._class_votes.pop(equip_id, None)
|
| 1202 |
-
logger.info(
|
| 1203 |
-
f"Class correction: {equip_id} ({CLASS_NAMES.get(current_class)}) → "
|
| 1204 |
-
f"{new_equip_id} ({CLASS_NAMES.get(winner)}) after {len(votes)} votes"
|
| 1205 |
-
)
|
| 1206 |
-
else:
|
| 1207 |
-
# Same class confirmed — hard-lock it
|
| 1208 |
-
self._class_locked[equip_id] = True
|
| 1209 |
-
self._class_votes.pop(equip_id, None)
|
| 1210 |
-
logger.info(f"{equip_id} class confirmed and hard-locked after {len(votes)} votes")
|
| 1211 |
-
|
| 1212 |
-
def is_active(self, equip_id: str, exclude_track: int = None,
|
| 1213 |
-
active_track_ids: set = None) -> bool:
|
| 1214 |
-
"""Check if an equipment ID is currently assigned to any active track.
|
| 1215 |
-
active_track_ids: set of track IDs present in the current frame.
|
| 1216 |
-
Only these are truly 'active' — gallery tracks are not."""
|
| 1217 |
-
if active_track_ids is None:
|
| 1218 |
-
return False # Can't determine without active set
|
| 1219 |
-
for tid, eid in self._track_to_equip.items():
|
| 1220 |
-
if eid == equip_id and tid != exclude_track and tid in active_track_ids:
|
| 1221 |
-
return True
|
| 1222 |
-
return False
|
| 1223 |
-
|
| 1224 |
-
def remove(self, track_id: int):
|
| 1225 |
-
self._track_to_equip.pop(track_id, None)
|
| 1226 |
-
|
| 1227 |
-
|
| 1228 |
-
# =============================================================================
|
| 1229 |
-
# Kafka Telemetry Producer
|
| 1230 |
-
# =============================================================================
|
| 1231 |
-
class TelemetryProducer:
|
| 1232 |
-
"""Produces equipment telemetry events to Kafka."""
|
| 1233 |
-
|
| 1234 |
-
def __init__(self, bootstrap_servers: str, topic: str):
|
| 1235 |
-
self.topic = topic
|
| 1236 |
-
self._producer = None
|
| 1237 |
-
self._bootstrap_servers = bootstrap_servers
|
| 1238 |
-
|
| 1239 |
-
def connect(self):
|
| 1240 |
-
from confluent_kafka import Producer
|
| 1241 |
-
self._producer = Producer({
|
| 1242 |
-
'bootstrap.servers': self._bootstrap_servers,
|
| 1243 |
-
'enable.idempotence': True,
|
| 1244 |
-
'acks': 'all',
|
| 1245 |
-
'linger.ms': 5,
|
| 1246 |
-
'batch.num.messages': 100,
|
| 1247 |
-
})
|
| 1248 |
-
logger.info(f"Kafka producer connected: {self._bootstrap_servers}")
|
| 1249 |
-
|
| 1250 |
-
def produce(self, equipment_id: str, payload: dict):
|
| 1251 |
-
"""Produce a single telemetry event, keyed by equipment_id."""
|
| 1252 |
-
if self._producer is None:
|
| 1253 |
-
return
|
| 1254 |
-
|
| 1255 |
-
self._producer.produce(
|
| 1256 |
-
topic=self.topic,
|
| 1257 |
-
key=equipment_id.encode('utf-8'),
|
| 1258 |
-
value=json.dumps(payload).encode('utf-8'),
|
| 1259 |
-
callback=self._delivery_callback
|
| 1260 |
-
)
|
| 1261 |
-
self._producer.poll(0)
|
| 1262 |
-
|
| 1263 |
-
def flush(self):
|
| 1264 |
-
if self._producer:
|
| 1265 |
-
self._producer.flush(timeout=5.0)
|
| 1266 |
-
|
| 1267 |
-
@staticmethod
|
| 1268 |
-
def _delivery_callback(err, msg):
|
| 1269 |
-
if err:
|
| 1270 |
-
logger.error(f"Kafka delivery failed: {err}")
|
| 1271 |
-
|
| 1272 |
-
|
| 1273 |
-
# =============================================================================
|
| 1274 |
-
# Main Pipeline Orchestrator
|
| 1275 |
-
# =============================================================================
|
| 1276 |
-
class InferencePipeline:
|
| 1277 |
-
"""
|
| 1278 |
-
Main pipeline class that orchestrates all CV components.
|
| 1279 |
-
|
| 1280 |
-
Flow: Frame → Detect → TAI Filter → Track → Re-ID → Motion → Activity → Kafka
|
| 1281 |
-
"""
|
| 1282 |
-
|
| 1283 |
-
def __init__(self, config: dict):
|
| 1284 |
-
self.config = config
|
| 1285 |
-
|
| 1286 |
-
# Core ML components (initialized in .initialize())
|
| 1287 |
-
self.detector = None
|
| 1288 |
-
self.tracker = None
|
| 1289 |
-
self.reid_gallery = None
|
| 1290 |
-
|
| 1291 |
-
# Analysis components
|
| 1292 |
-
self.motion_detector = ArticulatedMotionDetector(
|
| 1293 |
-
grid_size=3,
|
| 1294 |
-
motion_threshold=config.get('motion_threshold', 0.5) # Audit #3: match class default
|
| 1295 |
-
)
|
| 1296 |
-
# Activity classification is handled by Phase 10 (spatial context).
|
| 1297 |
-
# RuleBasedActivityClassifier kept for remove_track cleanup only.
|
| 1298 |
-
self.activity_classifier = RuleBasedActivityClassifier()
|
| 1299 |
-
|
| 1300 |
-
# Option B: Video-based activity classifier (X3D-S)
|
| 1301 |
-
# Falls back to rule-based Phase 10 if weights not available.
|
| 1302 |
-
self.video_classifier = VideoActivityClassifier(
|
| 1303 |
-
weights_path=config.get('activity_classifier_weights', 'weights/activity_classifier_x3d_s.pt'),
|
| 1304 |
-
clip_length=16,
|
| 1305 |
-
clip_stride=8,
|
| 1306 |
-
crop_size=224,
|
| 1307 |
-
confidence_threshold=0.5,
|
| 1308 |
-
device='cuda',
|
| 1309 |
-
)
|
| 1310 |
-
|
| 1311 |
-
self.id_generator = EquipmentIDGenerator()
|
| 1312 |
-
self.telemetry_producer = TelemetryProducer(
|
| 1313 |
-
bootstrap_servers=config.get('kafka_servers', 'kafka:9092'),
|
| 1314 |
-
topic=config.get('kafka_topic', 'equipment.telemetry.raw')
|
| 1315 |
-
)
|
| 1316 |
-
|
| 1317 |
-
# Frame counter for skip strategies
|
| 1318 |
-
self._frame_count = 0
|
| 1319 |
-
# Track IDs from previous frame (for detecting lost tracks)
|
| 1320 |
-
self._prev_track_ids = set()
|
| 1321 |
-
# Track first-seen frame for deferred ID creation.
|
| 1322 |
-
# Tracks must persist for MIN_TRACK_FRAMES before getting an equipment ID.
|
| 1323 |
-
# This prevents transient false detections from consuming ID numbers.
|
| 1324 |
-
self._track_first_seen = {} # track_id -> frame_id
|
| 1325 |
-
self.MIN_TRACK_FRAMES = 30 # 0.5s at 60fps
|
| 1326 |
-
# Activity smoothing
|
| 1327 |
-
self._activity_history = {} # equip_id -> deque(...)
|
| 1328 |
-
|
| 1329 |
-
def initialize(self):
|
| 1330 |
-
"""Initialize all components."""
|
| 1331 |
-
logger.info("Initializing inference pipeline...")
|
| 1332 |
-
|
| 1333 |
-
import shutil
|
| 1334 |
-
model_dir = os.getenv('MODEL_PATH', '/models')
|
| 1335 |
-
|
| 1336 |
-
# Phase 3: RF-DETR Detector
|
| 1337 |
-
weights_path = resolve_weights('rfdetr_construction.pth', local_dir=model_dir)
|
| 1338 |
-
|
| 1339 |
-
# Prevent base model redownloads by copying the user mapped file
|
| 1340 |
-
user_model = os.path.join(model_dir, "rf-detr-base.pth")
|
| 1341 |
-
if os.path.exists(user_model) and not os.path.exists("rf-detr-base.pth"):
|
| 1342 |
-
try:
|
| 1343 |
-
shutil.copy2(user_model, "rf-detr-base.pth")
|
| 1344 |
-
logger.info("Found user's manually downloaded rf-detr-base.pth. Bypassing download!")
|
| 1345 |
-
except Exception as e:
|
| 1346 |
-
pass
|
| 1347 |
-
self.detector = RFDETRDetector(
|
| 1348 |
-
weights_path=weights_path,
|
| 1349 |
-
confidence_threshold=self.config.get('confidence_threshold', 0.35),
|
| 1350 |
-
device=self.config.get('device', 'cuda')
|
| 1351 |
-
)
|
| 1352 |
-
self.detector.load()
|
| 1353 |
-
|
| 1354 |
-
# Phase 4: BoT-SORT Tracker
|
| 1355 |
-
osnet_path = resolve_weights('osnet_x0_25_msmt17.pt', local_dir=model_dir)
|
| 1356 |
-
|
| 1357 |
-
# Copy to boxmot's expected location to avoid re-downloading
|
| 1358 |
-
reid_dst = "/usr/local/lib/python3.10/dist-packages/models/osnet_x0_25_msmt17.pt"
|
| 1359 |
-
if os.path.exists(osnet_path) and not os.path.exists(reid_dst):
|
| 1360 |
-
os.makedirs(os.path.dirname(reid_dst), exist_ok=True)
|
| 1361 |
-
shutil.copy2(osnet_path, reid_dst)
|
| 1362 |
-
logger.info("Copied cached ReID weights — skipping download")
|
| 1363 |
-
|
| 1364 |
-
from boxmot import create_tracker
|
| 1365 |
-
botsort_config = Path("/configs/botsort.yaml")
|
| 1366 |
-
self.tracker = create_tracker(
|
| 1367 |
-
'botsort',
|
| 1368 |
-
tracker_config=botsort_config,
|
| 1369 |
-
reid_weights=Path(reid_dst if os.path.exists(reid_dst) else osnet_path),
|
| 1370 |
-
device='0',
|
| 1371 |
-
half=True
|
| 1372 |
-
)
|
| 1373 |
-
# Log actual tracker params to verify config was parsed correctly
|
| 1374 |
-
max_time_lost = getattr(self.tracker, 'max_time_lost', '?')
|
| 1375 |
-
buffer_size = getattr(self.tracker, 'buffer_size', '?')
|
| 1376 |
-
logger.info(
|
| 1377 |
-
f"BoT-SORT initialized: max_time_lost={max_time_lost}, "
|
| 1378 |
-
f"buffer_size={buffer_size}"
|
| 1379 |
-
)
|
| 1380 |
-
|
| 1381 |
-
# Phase 5: DINOv3 Re-ID Gallery
|
| 1382 |
-
self.reid_gallery = DINOv3ReIDGallery(
|
| 1383 |
-
similarity_threshold=self.config.get('reid_threshold', 0.75),
|
| 1384 |
-
gallery_ttl_frames=self.config.get('reid_gallery_ttl', 1800),
|
| 1385 |
-
device=self.config.get('device', 'cuda'),
|
| 1386 |
-
spatial_gate_ratio=0.60,
|
| 1387 |
-
)
|
| 1388 |
-
self.reid_gallery.load()
|
| 1389 |
-
|
| 1390 |
-
# Option B: Video activity classifier (X3D-S)
|
| 1391 |
-
if self.video_classifier.load():
|
| 1392 |
-
logger.info("Video activity classifier loaded — will override Phase 10 rules")
|
| 1393 |
-
else:
|
| 1394 |
-
logger.info("Video activity classifier not available — using rule-based Phase 10")
|
| 1395 |
-
|
| 1396 |
-
# Kafka producer
|
| 1397 |
-
self.telemetry_producer.connect()
|
| 1398 |
-
|
| 1399 |
-
logger.info("Pipeline initialized — all components ready.")
|
| 1400 |
-
|
| 1401 |
-
def process_frame(self, frame: np.ndarray, frame_id: int,
|
| 1402 |
-
timestamp: float, source_id: str = "cam_01",
|
| 1403 |
-
fps: float = 30.0):
|
| 1404 |
-
"""
|
| 1405 |
-
Process a single frame through the full pipeline.
|
| 1406 |
-
|
| 1407 |
-
Args:
|
| 1408 |
-
frame: BGR image (H, W, 3)
|
| 1409 |
-
frame_id: Frame counter
|
| 1410 |
-
|
| 1411 |
-
Returns:
|
| 1412 |
-
List of annotation dicts for visualization (empty if no tracks)
|
| 1413 |
-
timestamp: Seconds from video start
|
| 1414 |
-
source_id: Camera identifier
|
| 1415 |
-
"""
|
| 1416 |
-
self._frame_count += 1
|
| 1417 |
-
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
| 1418 |
-
|
| 1419 |
-
# ----- Phase 3: Detection -----
|
| 1420 |
-
raw_detections = self.detector.predict(frame)
|
| 1421 |
-
|
| 1422 |
-
if len(raw_detections) == 0:
|
| 1423 |
-
return []
|
| 1424 |
-
|
| 1425 |
-
# ----- Phase 4: TAI Filter -----
|
| 1426 |
-
# Audit #8: wrap in try/except — active_tracks API varies across BoxMOT versions
|
| 1427 |
-
tracked_boxes = np.empty((0, 4))
|
| 1428 |
-
try:
|
| 1429 |
-
if hasattr(self.tracker, 'active_tracks') and len(self.tracker.active_tracks) > 0:
|
| 1430 |
-
tracked_boxes = np.array([t.xyxy for t in self.tracker.active_tracks])
|
| 1431 |
-
except (AttributeError, TypeError):
|
| 1432 |
-
pass # Graceful fallback — TAI filter runs without tracked boxes
|
| 1433 |
-
|
| 1434 |
-
clean_detections = apply_tai_heuristic(
|
| 1435 |
-
raw_detections, tracked_boxes, ioa_threshold=0.4
|
| 1436 |
-
)
|
| 1437 |
-
|
| 1438 |
-
# ----- Min-area filter: remove tiny background objects -----
|
| 1439 |
-
# Construction equipment from aerial/drone footage has a minimum
|
| 1440 |
-
# pixel footprint. Distant vehicles (like the background blue truck)
|
| 1441 |
-
# are too small to be real tracked equipment.
|
| 1442 |
-
if len(clean_detections) > 0:
|
| 1443 |
-
frame_area = frame.shape[0] * frame.shape[1]
|
| 1444 |
-
min_area_ratio = 0.003 # 0.3% of frame area (~6200px at 1080p)
|
| 1445 |
-
areas = (clean_detections[:, 2] - clean_detections[:, 0]) * \
|
| 1446 |
-
(clean_detections[:, 3] - clean_detections[:, 1])
|
| 1447 |
-
area_mask = areas >= (frame_area * min_area_ratio)
|
| 1448 |
-
clean_detections = clean_detections[area_mask]
|
| 1449 |
-
|
| 1450 |
-
# ----- Phase 4: BoT-SORT Tracking -----
|
| 1451 |
-
tracked_objects = self.tracker.update(clean_detections, frame)
|
| 1452 |
-
|
| 1453 |
-
if self._frame_count % 30 == 0:
|
| 1454 |
-
logger.info(f"Frame {frame_id}: Dets={len(raw_detections)}, Clean={len(clean_detections)}, Tracked={len(tracked_objects)}")
|
| 1455 |
-
|
| 1456 |
-
# ----- Phase 5: Re-ID — detect lost tracks -----
|
| 1457 |
-
current_track_ids = set()
|
| 1458 |
-
if len(tracked_objects) > 0:
|
| 1459 |
-
current_track_ids = {int(t[4]) for t in tracked_objects}
|
| 1460 |
-
|
| 1461 |
-
lost_ids = self._prev_track_ids - current_track_ids
|
| 1462 |
-
new_track_ids = current_track_ids - self._prev_track_ids
|
| 1463 |
-
|
| 1464 |
-
for lost_id in lost_ids:
|
| 1465 |
-
self.reid_gallery.move_to_gallery(lost_id, frame_id)
|
| 1466 |
-
self.motion_detector.remove_track(f"pending_{lost_id}")
|
| 1467 |
-
self.activity_classifier.remove_track(f"pending_{lost_id}")
|
| 1468 |
-
self.video_classifier.remove_track(f"pending_{lost_id}")
|
| 1469 |
-
self._track_first_seen.pop(lost_id, None)
|
| 1470 |
-
|
| 1471 |
-
# ----- Process each tracked object -----
|
| 1472 |
-
annotations = []
|
| 1473 |
-
for track in tracked_objects:
|
| 1474 |
-
bbox = track[:4]
|
| 1475 |
-
track_id = int(track[4])
|
| 1476 |
-
conf = float(track[5])
|
| 1477 |
-
class_id = int(track[6])
|
| 1478 |
-
|
| 1479 |
-
# Record first-seen frame for deferred ID creation
|
| 1480 |
-
if track_id not in self._track_first_seen:
|
| 1481 |
-
self._track_first_seen[track_id] = frame_id
|
| 1482 |
-
|
| 1483 |
-
track_age = frame_id - self._track_first_seen[track_id]
|
| 1484 |
-
equip_id = self.id_generator.get_existing(track_id)
|
| 1485 |
-
|
| 1486 |
-
# ----- Phase 5: Re-ID + Equipment ID assignment -----
|
| 1487 |
-
if track_id in new_track_ids or self._frame_count <= 1:
|
| 1488 |
-
# New track — attempt Re-ID immediately
|
| 1489 |
-
embedding = self.reid_gallery.extract_embedding(frame, bbox)
|
| 1490 |
-
color_hist = self.reid_gallery._extract_color_histogram(frame, bbox)
|
| 1491 |
-
matched_id = None
|
| 1492 |
-
|
| 1493 |
-
if equip_id is None and embedding is not None:
|
| 1494 |
-
matched_id, sim = self.reid_gallery.query(
|
| 1495 |
-
embedding, class_id, frame_id,
|
| 1496 |
-
query_bbox=bbox,
|
| 1497 |
-
frame_width=frame.shape[1],
|
| 1498 |
-
frame_height=frame.shape[0],
|
| 1499 |
-
query_color_hist=color_hist
|
| 1500 |
-
)
|
| 1501 |
-
if matched_id is not None:
|
| 1502 |
-
if self.id_generator.is_active(matched_id, exclude_track=track_id, active_track_ids=current_track_ids):
|
| 1503 |
-
logger.info(f"Re-ID rejected: {matched_id} already active on another track")
|
| 1504 |
-
matched_id = None
|
| 1505 |
-
else:
|
| 1506 |
-
equip_id = matched_id
|
| 1507 |
-
self.id_generator.reassign(track_id, equip_id)
|
| 1508 |
-
locked = self.id_generator.get_locked_class(equip_id)
|
| 1509 |
-
if locked is not None:
|
| 1510 |
-
class_id = locked
|
| 1511 |
-
|
| 1512 |
-
if embedding is not None:
|
| 1513 |
-
self.reid_gallery.update_active(
|
| 1514 |
-
track_id, embedding, equip_id, class_id,
|
| 1515 |
-
bbox, frame_id
|
| 1516 |
-
)
|
| 1517 |
-
if color_hist is not None and track_id in self.reid_gallery._active_embeddings:
|
| 1518 |
-
self.reid_gallery._active_embeddings[track_id]['color_hist'] = color_hist
|
| 1519 |
-
|
| 1520 |
-
if equip_id is not None:
|
| 1521 |
-
locked_class = self.id_generator.get_locked_class(equip_id)
|
| 1522 |
-
if locked_class is not None:
|
| 1523 |
-
class_id = locked_class
|
| 1524 |
-
else:
|
| 1525 |
-
continue # Pending — skip annotation
|
| 1526 |
-
|
| 1527 |
-
elif equip_id is None:
|
| 1528 |
-
# Existing track, no equipment ID yet — check if mature enough
|
| 1529 |
-
if track_age >= self.MIN_TRACK_FRAMES:
|
| 1530 |
-
embedding = self.reid_gallery.extract_embedding(frame, bbox)
|
| 1531 |
-
color_hist = self.reid_gallery._extract_color_histogram(frame, bbox)
|
| 1532 |
-
matched_id = None
|
| 1533 |
-
|
| 1534 |
-
if embedding is not None:
|
| 1535 |
-
matched_id, sim = self.reid_gallery.query(
|
| 1536 |
-
embedding, class_id, frame_id,
|
| 1537 |
-
query_bbox=bbox,
|
| 1538 |
-
frame_width=frame.shape[1],
|
| 1539 |
-
frame_height=frame.shape[0],
|
| 1540 |
-
query_color_hist=color_hist
|
| 1541 |
-
)
|
| 1542 |
-
if matched_id is not None:
|
| 1543 |
-
if self.id_generator.is_active(matched_id, exclude_track=track_id, active_track_ids=current_track_ids):
|
| 1544 |
-
logger.info(f"Re-ID rejected: {matched_id} already active on another track")
|
| 1545 |
-
matched_id = None
|
| 1546 |
-
else:
|
| 1547 |
-
equip_id = matched_id
|
| 1548 |
-
self.id_generator.reassign(track_id, equip_id)
|
| 1549 |
-
|
| 1550 |
-
if equip_id is None:
|
| 1551 |
-
equip_id = self.id_generator.get_or_create(track_id, class_id, conf=conf)
|
| 1552 |
-
logger.info(f"Track {track_id} confirmed after {track_age} frames → {equip_id}")
|
| 1553 |
-
|
| 1554 |
-
locked_class = self.id_generator.get_locked_class(equip_id)
|
| 1555 |
-
if locked_class is not None:
|
| 1556 |
-
class_id = locked_class
|
| 1557 |
-
|
| 1558 |
-
if embedding is not None:
|
| 1559 |
-
self.reid_gallery.update_active(
|
| 1560 |
-
track_id, embedding, equip_id, class_id,
|
| 1561 |
-
bbox, frame_id
|
| 1562 |
-
)
|
| 1563 |
-
if color_hist is not None and track_id in self.reid_gallery._active_embeddings:
|
| 1564 |
-
self.reid_gallery._active_embeddings[track_id]['color_hist'] = color_hist
|
| 1565 |
-
else:
|
| 1566 |
-
if self._frame_count % 10 == 0:
|
| 1567 |
-
embedding = self.reid_gallery.extract_embedding(frame, bbox)
|
| 1568 |
-
if embedding is not None:
|
| 1569 |
-
self.reid_gallery.update_active(
|
| 1570 |
-
track_id, embedding, None, class_id,
|
| 1571 |
-
bbox, frame_id
|
| 1572 |
-
)
|
| 1573 |
-
continue # Still pending
|
| 1574 |
-
|
| 1575 |
-
else:
|
| 1576 |
-
# Existing track WITH equipment ID — normal path
|
| 1577 |
-
if not self.id_generator.is_class_locked(equip_id):
|
| 1578 |
-
self.id_generator.vote_class(track_id, equip_id, class_id, conf)
|
| 1579 |
-
equip_id = self.id_generator.get_existing(track_id)
|
| 1580 |
-
locked_class = self.id_generator.get_locked_class(equip_id)
|
| 1581 |
-
if locked_class is not None:
|
| 1582 |
-
class_id = locked_class
|
| 1583 |
-
if self._frame_count % 30 == 0:
|
| 1584 |
-
embedding = self.reid_gallery.extract_embedding(frame, bbox)
|
| 1585 |
-
if embedding is not None:
|
| 1586 |
-
self.reid_gallery.update_active(
|
| 1587 |
-
track_id, embedding, equip_id, class_id,
|
| 1588 |
-
bbox, frame_id
|
| 1589 |
-
)
|
| 1590 |
-
color_hist = self.reid_gallery._extract_color_histogram(frame, bbox)
|
| 1591 |
-
if color_hist is not None and track_id in self.reid_gallery._active_embeddings:
|
| 1592 |
-
self.reid_gallery._active_embeddings[track_id]['color_hist'] = color_hist
|
| 1593 |
-
|
| 1594 |
-
stable_id = equip_id if equip_id is not None else f"pending_{track_id}"
|
| 1595 |
-
|
| 1596 |
-
# ----- Phase 6: Articulated Motion -----
|
| 1597 |
-
motion_result = self.motion_detector.detect(
|
| 1598 |
-
frame_gray, stable_id, bbox
|
| 1599 |
-
)
|
| 1600 |
-
|
| 1601 |
-
# ----- Phase 7: Pass through raw motion state -----
|
| 1602 |
-
# Phase 10 owns ALL activity classification using spatial context.
|
| 1603 |
-
# Phase 7 only records the motion detector's ACTIVE/INACTIVE decision
|
| 1604 |
-
# and motion_source — no activity sub-labels assigned here.
|
| 1605 |
-
equip_class = self._class_name(class_id)
|
| 1606 |
-
is_active = motion_result['is_active']
|
| 1607 |
-
|
| 1608 |
-
# Collect annotation — state/activity assigned by Phase 10
|
| 1609 |
-
annotations.append({
|
| 1610 |
-
'bbox': bbox,
|
| 1611 |
-
'equip_id': equip_id,
|
| 1612 |
-
'stable_id': stable_id,
|
| 1613 |
-
'class_name': equip_class,
|
| 1614 |
-
'conf': conf,
|
| 1615 |
-
'is_motion_active': is_active, # raw from motion detector
|
| 1616 |
-
'motion_source': motion_result['motion_source'],
|
| 1617 |
-
'cell_motions': motion_result['cell_motions'], # 3x3 grid for bucket height
|
| 1618 |
-
'state': '', # Phase 10 assigns
|
| 1619 |
-
'activity': '', # Phase 10 assigns
|
| 1620 |
-
'frame_id': frame_id,
|
| 1621 |
-
'timestamp': f"{int(timestamp // 3600):02d}:{int((timestamp % 3600) // 60):02d}:{timestamp % 60:06.3f}",
|
| 1622 |
-
'source_id': source_id,
|
| 1623 |
-
'track_id': track_id, # needed for video classifier lookup
|
| 1624 |
-
})
|
| 1625 |
-
|
| 1626 |
-
# Feed crop to video activity classifier buffer
|
| 1627 |
-
if self.video_classifier.is_loaded:
|
| 1628 |
-
x1, y1, x2, y2 = [int(v) for v in bbox[:4]]
|
| 1629 |
-
x1, y1 = max(0, x1), max(0, y1)
|
| 1630 |
-
x2 = min(frame.shape[1], x2)
|
| 1631 |
-
y2 = min(frame.shape[0], y2)
|
| 1632 |
-
crop = frame[y1:y2, x1:x2]
|
| 1633 |
-
self.video_classifier.feed_crop(stable_id, crop)
|
| 1634 |
-
|
| 1635 |
-
# ----- Phase 10: Activity Classification -----
|
| 1636 |
-
# Single source of truth for ALL state + activity labels.
|
| 1637 |
-
#
|
| 1638 |
-
# Uses two signals from the 3x3 optical flow grid:
|
| 1639 |
-
# 1. Bucket height: flow concentrated in TOP rows = bucket elevated (DUMPING)
|
| 1640 |
-
# flow concentrated in BOTTOM rows = bucket low (DIGGING)
|
| 1641 |
-
# 2. Motion source: whole_body = machine translating (LOADING/swinging)
|
| 1642 |
-
# partial = only part moving (bucket work)
|
| 1643 |
-
#
|
| 1644 |
-
# Decision table for articulated equipment:
|
| 1645 |
-
# INACTIVE → WAITING
|
| 1646 |
-
# ACTIVE + whole_body → LOADING (swinging/translating)
|
| 1647 |
-
# ACTIVE + partial + flow in top rows → DUMPING (bucket elevated)
|
| 1648 |
-
# ACTIVE + partial + flow in bottom rows → DIGGING (bucket low, scooping)
|
| 1649 |
-
#
|
| 1650 |
-
# Dump trucks: classified via adjacent active articulated equipment.
|
| 1651 |
-
|
| 1652 |
-
ARTICULATED = {'excavator', 'wheel_loader', 'bulldozer'}
|
| 1653 |
-
|
| 1654 |
-
def bucket_height_score(cell_motions):
|
| 1655 |
-
"""
|
| 1656 |
-
Compute where the flow is concentrated vertically in the 3x3 grid.
|
| 1657 |
-
Returns a score: positive = top rows (bucket high), negative = bottom rows (bucket low).
|
| 1658 |
-
|
| 1659 |
-
Grid layout (3x3):
|
| 1660 |
-
row 0 = top of bbox (bucket elevated / dumping position)
|
| 1661 |
-
row 1 = middle
|
| 1662 |
-
row 2 = bottom of bbox (bucket low / digging position)
|
| 1663 |
-
"""
|
| 1664 |
-
if not cell_motions or len(cell_motions) < 3:
|
| 1665 |
-
return 0.0
|
| 1666 |
-
# Sum flow magnitude across each row (all 3 columns)
|
| 1667 |
-
top_flow = sum(cell_motions[0]) # row 0
|
| 1668 |
-
mid_flow = sum(cell_motions[1]) # row 1
|
| 1669 |
-
bottom_flow = sum(cell_motions[2]) # row 2
|
| 1670 |
-
total = top_flow + mid_flow + bottom_flow
|
| 1671 |
-
if total < 0.01:
|
| 1672 |
-
return 0.0
|
| 1673 |
-
# Score: +1 = all flow in top, -1 = all flow in bottom
|
| 1674 |
-
return (top_flow - bottom_flow) / total
|
| 1675 |
-
|
| 1676 |
-
def bboxes_overlap(a, b):
|
| 1677 |
-
x_overlap = min(a[2], b[2]) - max(a[0], b[0])
|
| 1678 |
-
y_overlap = min(a[3], b[3]) - max(a[1], b[1])
|
| 1679 |
-
return x_overlap > 0 and y_overlap > 0
|
| 1680 |
-
|
| 1681 |
-
# --- Pass 1: Classify articulated equipment ---
|
| 1682 |
-
# If X3D-S video classifier is loaded, use it (Option B).
|
| 1683 |
-
# Otherwise, fall back to bucket height heuristic (Option A).
|
| 1684 |
-
use_video_classifier = self.video_classifier.is_loaded
|
| 1685 |
-
|
| 1686 |
-
# Calculate smoothing window dynamically (0.50 seconds to reduce lag)
|
| 1687 |
-
window_size = max(10, int(fps * 0.50))
|
| 1688 |
-
|
| 1689 |
-
for ann in annotations:
|
| 1690 |
-
if ann is None or ann['class_name'] not in ARTICULATED:
|
| 1691 |
-
continue
|
| 1692 |
-
|
| 1693 |
-
# Get video classifier output if available and motion is active
|
| 1694 |
-
if use_video_classifier and ann['is_motion_active']:
|
| 1695 |
-
activity, conf = self.video_classifier.classify(ann['stable_id'])
|
| 1696 |
-
if activity is not None:
|
| 1697 |
-
ann['state'] = 'ACTIVE'
|
| 1698 |
-
ann['activity'] = activity
|
| 1699 |
-
ann['activity_conf'] = conf
|
| 1700 |
-
continue
|
| 1701 |
-
|
| 1702 |
-
# Fallback: rule-based Option A
|
| 1703 |
-
raw_activity = 'WAITING'
|
| 1704 |
-
if ann['is_motion_active']:
|
| 1705 |
-
score = bucket_height_score(ann['cell_motions'])
|
| 1706 |
-
if ann['motion_source'] == 'whole_body':
|
| 1707 |
-
if score < -0.15:
|
| 1708 |
-
raw_activity = 'DIGGING'
|
| 1709 |
-
else:
|
| 1710 |
-
raw_activity = 'LOADING'
|
| 1711 |
-
else:
|
| 1712 |
-
if score > 0.1:
|
| 1713 |
-
raw_activity = 'DUMPING'
|
| 1714 |
-
else:
|
| 1715 |
-
raw_activity = 'DIGGING'
|
| 1716 |
-
|
| 1717 |
-
# Apply temporal smoothing for Option A using equip_id
|
| 1718 |
-
eid = ann['equip_id']
|
| 1719 |
-
if eid not in self._activity_history:
|
| 1720 |
-
self._activity_history[eid] = deque(maxlen=window_size)
|
| 1721 |
-
|
| 1722 |
-
if self._activity_history[eid].maxlen != window_size:
|
| 1723 |
-
old_items = list(self._activity_history[eid])
|
| 1724 |
-
self._activity_history[eid] = deque(old_items[-window_size:], maxlen=window_size)
|
| 1725 |
-
|
| 1726 |
-
self._activity_history[eid].append(raw_activity)
|
| 1727 |
-
|
| 1728 |
-
# Majority vote (protect against plurality vote splitting active states)
|
| 1729 |
-
from collections import Counter
|
| 1730 |
-
counts = Counter(self._activity_history[eid])
|
| 1731 |
-
waiting_votes = counts.get('WAITING', 0)
|
| 1732 |
-
active_votes = sum(c for s, c in counts.items() if s != 'WAITING')
|
| 1733 |
-
|
| 1734 |
-
if waiting_votes > active_votes:
|
| 1735 |
-
ann['activity'] = 'WAITING'
|
| 1736 |
-
ann['state'] = 'INACTIVE'
|
| 1737 |
-
else:
|
| 1738 |
-
active_only = {s: c for s, c in counts.items() if s != 'WAITING'}
|
| 1739 |
-
ann['activity'] = max(active_only.items(), key=lambda x: x[1])[0] if active_only else 'LOADING'
|
| 1740 |
-
ann['state'] = 'ACTIVE'
|
| 1741 |
-
|
| 1742 |
-
# --- Pass 2: Classify dump trucks via adjacent active articulated ---
|
| 1743 |
-
dt_annotations = [a for a in annotations if a is not None and a['class_name'] == 'dump_truck']
|
| 1744 |
-
active_articulateds = [
|
| 1745 |
-
a for a in annotations
|
| 1746 |
-
if a is not None
|
| 1747 |
-
and a['state'] == 'ACTIVE'
|
| 1748 |
-
and a['class_name'] in ARTICULATED
|
| 1749 |
-
]
|
| 1750 |
-
for dt in dt_annotations:
|
| 1751 |
-
d_bbox = dt['bbox'][:4]
|
| 1752 |
-
near_loader = False
|
| 1753 |
-
for loader in active_articulateds:
|
| 1754 |
-
if bboxes_overlap(d_bbox, loader['bbox'][:4]):
|
| 1755 |
-
near_loader = True
|
| 1756 |
-
break
|
| 1757 |
-
|
| 1758 |
-
if near_loader:
|
| 1759 |
-
dt['state'] = 'ACTIVE'
|
| 1760 |
-
dt['activity'] = 'LOADING'
|
| 1761 |
-
elif dt['is_motion_active']:
|
| 1762 |
-
# Truck is moving — map to default ACTIVE (LOADING) since HAULING is not in the 4-activity schema!
|
| 1763 |
-
dt['state'] = 'ACTIVE'
|
| 1764 |
-
dt['activity'] = 'LOADING'
|
| 1765 |
-
else:
|
| 1766 |
-
dt['state'] = 'INACTIVE'
|
| 1767 |
-
dt['activity'] = 'WAITING'
|
| 1768 |
-
|
| 1769 |
-
# --- Pass 3: Default for any remaining equipment (cranes, etc.) ---
|
| 1770 |
-
for ann in annotations:
|
| 1771 |
-
if ann is None or ann['state'] != '':
|
| 1772 |
-
continue
|
| 1773 |
-
if ann['is_motion_active']:
|
| 1774 |
-
ann['state'] = 'ACTIVE'
|
| 1775 |
-
ann['activity'] = 'LOADING'
|
| 1776 |
-
else:
|
| 1777 |
-
ann['state'] = 'INACTIVE'
|
| 1778 |
-
ann['activity'] = 'WAITING'
|
| 1779 |
-
|
| 1780 |
-
# Debug: log Phase 10 decisions once per second
|
| 1781 |
-
if frame_id % 60 == 0:
|
| 1782 |
-
for ann in annotations:
|
| 1783 |
-
if ann is None:
|
| 1784 |
-
continue
|
| 1785 |
-
bh = bucket_height_score(ann.get('cell_motions', []))
|
| 1786 |
-
src = 'x3d' if ann.get('activity_conf') else 'rules'
|
| 1787 |
-
conf_str = f" ({ann['activity_conf']:.2f})" if ann.get('activity_conf') else ''
|
| 1788 |
-
logger.info(
|
| 1789 |
-
f"Phase10[{ann['equip_id']}]: class={ann['class_name']}, "
|
| 1790 |
-
f"motion_active={ann['is_motion_active']}, "
|
| 1791 |
-
f"motion_src={ann['motion_source']}, bucket_h={bh:+.2f}, "
|
| 1792 |
-
f"→ {ann['state']} | {ann['activity']}{conf_str} [{src}]"
|
| 1793 |
-
)
|
| 1794 |
-
|
| 1795 |
-
# ----- Deduplicate equipment IDs -----
|
| 1796 |
-
# If two tracks share the same equipment ID (e.g., WL-002 on both the
|
| 1797 |
-
# real wheel loader and a false detection), keep only the higher-confidence
|
| 1798 |
-
# annotation. The lower-confidence duplicate is dropped from display.
|
| 1799 |
-
seen_ids = {}
|
| 1800 |
-
for i, ann in enumerate(annotations):
|
| 1801 |
-
eid = ann['equip_id']
|
| 1802 |
-
if eid in seen_ids:
|
| 1803 |
-
prev_idx = seen_ids[eid]
|
| 1804 |
-
if ann['conf'] > annotations[prev_idx]['conf']:
|
| 1805 |
-
annotations[prev_idx] = None # mark old for removal
|
| 1806 |
-
seen_ids[eid] = i
|
| 1807 |
-
else:
|
| 1808 |
-
annotations[i] = None # mark new for removal
|
| 1809 |
-
else:
|
| 1810 |
-
seen_ids[eid] = i
|
| 1811 |
-
annotations = [a for a in annotations if a is not None]
|
| 1812 |
-
|
| 1813 |
-
# ----- Phase 11: Produce to Kafka (after spatial context + dedup) -----
|
| 1814 |
-
for ann in annotations:
|
| 1815 |
-
payload = {
|
| 1816 |
-
'frame_id': ann['frame_id'],
|
| 1817 |
-
'equipment_id': ann['equip_id'],
|
| 1818 |
-
'equipment_class': ann['class_name'],
|
| 1819 |
-
'timestamp': ann['timestamp'],
|
| 1820 |
-
'bbox': {
|
| 1821 |
-
'x1': float(ann['bbox'][0]), 'y1': float(ann['bbox'][1]),
|
| 1822 |
-
'x2': float(ann['bbox'][2]), 'y2': float(ann['bbox'][3])
|
| 1823 |
-
},
|
| 1824 |
-
'detection_confidence': round(ann['conf'], 3),
|
| 1825 |
-
'utilization': {
|
| 1826 |
-
'current_state': ann['state'],
|
| 1827 |
-
'current_activity': ann['activity'],
|
| 1828 |
-
'motion_source': ann['motion_source']
|
| 1829 |
-
},
|
| 1830 |
-
'source_id': ann['source_id'],
|
| 1831 |
-
'pipeline_timestamp': datetime.now(timezone.utc).isoformat()
|
| 1832 |
-
}
|
| 1833 |
-
self.telemetry_producer.produce(ann['equip_id'], payload)
|
| 1834 |
-
|
| 1835 |
-
# Periodic flush
|
| 1836 |
-
if self._frame_count % 30 == 0:
|
| 1837 |
-
self.telemetry_producer.flush()
|
| 1838 |
-
|
| 1839 |
-
# Periodic gallery cleanup — recycle expired phantom IDs (self-healing)
|
| 1840 |
-
if self._frame_count % 300 == 0: # Every 5 seconds at 60fps
|
| 1841 |
-
self.reid_gallery.cleanup(frame_id, id_generator=self.id_generator)
|
| 1842 |
-
|
| 1843 |
-
# Cleanup stale activity history
|
| 1844 |
-
active_known_eids = set(self.id_generator._equip_to_class.keys())
|
| 1845 |
-
stale_hist = [eid for eid in self._activity_history if eid not in active_known_eids]
|
| 1846 |
-
for eid in stale_hist:
|
| 1847 |
-
del self._activity_history[eid]
|
| 1848 |
-
|
| 1849 |
-
# Cleanup stale stable_id memories inside motion detector
|
| 1850 |
-
if hasattr(self.motion_detector, '_prev_crops'):
|
| 1851 |
-
stale_md = [eid for eid in self.motion_detector._prev_crops if not str(eid).startswith('pending_') and eid not in active_known_eids]
|
| 1852 |
-
for eid in stale_md:
|
| 1853 |
-
self.motion_detector.remove_track(eid)
|
| 1854 |
-
|
| 1855 |
-
# Cleanup stale stable_id memories inside video classifier
|
| 1856 |
-
if hasattr(self.video_classifier, '_crop_buffer'):
|
| 1857 |
-
stale_vc = [eid for eid in self.video_classifier._crop_buffer if not str(eid).startswith('pending_') and eid not in active_known_eids]
|
| 1858 |
-
for eid in stale_vc:
|
| 1859 |
-
self.video_classifier.remove_track(eid)
|
| 1860 |
-
|
| 1861 |
-
# Audit #2: update _prev_track_ids AFTER the loop so new_track_ids works next frame
|
| 1862 |
-
self._prev_track_ids = current_track_ids
|
| 1863 |
-
|
| 1864 |
-
return annotations
|
| 1865 |
-
|
| 1866 |
-
|
| 1867 |
-
@staticmethod
|
| 1868 |
-
def _class_name(class_id: int) -> str:
|
| 1869 |
-
return CLASS_NAMES.get(int(class_id), 'unknown')
|
| 1870 |
-
|
| 1871 |
-
@staticmethod
|
| 1872 |
-
def draw_annotations(frame: np.ndarray, annotations: list,
|
| 1873 |
-
frame_id: int, timestamp: float = 0.0,
|
| 1874 |
-
fps: float = 0.0) -> np.ndarray:
|
| 1875 |
-
"""
|
| 1876 |
-
Draw bounding boxes, labels, and overlay info on the frame.
|
| 1877 |
-
Returns annotated frame copy.
|
| 1878 |
-
"""
|
| 1879 |
-
vis = frame.copy()
|
| 1880 |
-
|
| 1881 |
-
# Color palette per class
|
| 1882 |
-
COLORS = {
|
| 1883 |
-
'excavator': (0, 200, 255), # Orange
|
| 1884 |
-
'dump_truck': (255, 180, 0), # Cyan-blue
|
| 1885 |
-
'bulldozer': (0, 255, 100), # Green
|
| 1886 |
-
'wheel_loader': (0, 255, 255), # Yellow
|
| 1887 |
-
'mobile_crane': (255, 0, 150), # Magenta
|
| 1888 |
-
'tower_crane': (150, 100, 255), # Purple
|
| 1889 |
-
'roller_compactor':(255, 100, 50), # Red-orange
|
| 1890 |
-
'cement_mixer': (100, 255, 200), # Teal
|
| 1891 |
-
'unknown': (200, 200, 200), # Gray
|
| 1892 |
-
}
|
| 1893 |
-
|
| 1894 |
-
for ann in annotations:
|
| 1895 |
-
x1, y1, x2, y2 = [int(v) for v in ann['bbox'][:4]]
|
| 1896 |
-
color = COLORS.get(ann['class_name'], (200, 200, 200))
|
| 1897 |
-
state_color = (0, 255, 0) if ann['state'] == 'ACTIVE' else (0, 0, 255)
|
| 1898 |
-
|
| 1899 |
-
# Draw bbox
|
| 1900 |
-
cv2.rectangle(vis, (x1, y1), (x2, y2), color, 2)
|
| 1901 |
-
|
| 1902 |
-
# State indicator bar (top of bbox)
|
| 1903 |
-
cv2.rectangle(vis, (x1, y1 - 4), (x2, y1), state_color, -1)
|
| 1904 |
-
|
| 1905 |
-
# Label background
|
| 1906 |
-
label = f"{ann['equip_id']} | {ann['class_name']} | {ann['conf']:.0%}"
|
| 1907 |
-
(tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
| 1908 |
-
cv2.rectangle(vis, (x1, y1 - 24), (x1 + tw + 6, y1 - 4), color, -1)
|
| 1909 |
-
cv2.putText(vis, label, (x1 + 3, y1 - 9),
|
| 1910 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
|
| 1911 |
-
|
| 1912 |
-
# Activity label below bbox
|
| 1913 |
-
act_label = f"{ann['state']} | {ann['activity']}"
|
| 1914 |
-
cv2.putText(vis, act_label, (x1, y2 + 16),
|
| 1915 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.45, state_color, 1, cv2.LINE_AA)
|
| 1916 |
-
|
| 1917 |
-
# HUD overlay with video timestamp
|
| 1918 |
-
mins = int(timestamp // 60)
|
| 1919 |
-
secs = timestamp % 60
|
| 1920 |
-
fps_str = f"{fps:.1f}" if fps >= 0 else "..."
|
| 1921 |
-
hud = f"{mins:02d}:{secs:05.2f} | Tracked: {len(annotations)} | FPS: {fps_str}"
|
| 1922 |
-
cv2.putText(vis, hud, (10, 25),
|
| 1923 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2, cv2.LINE_AA)
|
| 1924 |
-
|
| 1925 |
-
return vis
|
| 1926 |
-
|
| 1927 |
-
def shutdown(self):
|
| 1928 |
-
"""Clean shutdown."""
|
| 1929 |
-
self.telemetry_producer.flush()
|
| 1930 |
-
logger.info("Pipeline shutdown complete.")
|
| 1931 |
-
|
| 1932 |
-
|
| 1933 |
-
# =============================================================================
|
| 1934 |
-
# Entry Point
|
| 1935 |
-
# =============================================================================
|
| 1936 |
-
def main():
|
| 1937 |
-
logger.info("=" * 60)
|
| 1938 |
-
logger.info(" Construction Equipment Monitor - CV Inference")
|
| 1939 |
-
logger.info("=" * 60)
|
| 1940 |
-
|
| 1941 |
-
# cv2 is now imported at module level
|
| 1942 |
-
|
| 1943 |
-
config = {
|
| 1944 |
-
'kafka_servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'kafka:9092'),
|
| 1945 |
-
'kafka_topic': os.getenv('KAFKA_TOPIC_RAW', 'equipment.telemetry.raw'),
|
| 1946 |
-
'video_source': os.getenv('VIDEO_SOURCE', '/data/test_video.mp4'),
|
| 1947 |
-
'device': os.getenv('DEVICE', 'cuda'),
|
| 1948 |
-
'motion_threshold': float(os.getenv('MOTION_THRESHOLD', '0.5')), # Audit #3: was 2.0
|
| 1949 |
-
'frame_skip': int(os.getenv('FRAME_SKIP', '1')),
|
| 1950 |
-
'target_width': int(os.getenv('TARGET_WIDTH', '960')),
|
| 1951 |
-
'confidence_threshold': float(os.getenv('CONFIDENCE_THRESHOLD', '0.50')),
|
| 1952 |
-
'reid_threshold': float(os.getenv('REID_THRESHOLD', '0.85')), # Conservative: Re-ID only fires after BoT-SORT drops track (>20s lost)
|
| 1953 |
-
'reid_gallery_ttl': int(os.getenv('REID_GALLERY_TTL', '18000')), # 5min at 60fps — vehicles can leave and return
|
| 1954 |
-
'weights_path': os.path.join(os.getenv('MODEL_PATH', '/models'), 'rfdetr_construction.pth'),
|
| 1955 |
-
}
|
| 1956 |
-
|
| 1957 |
-
# Initialize pipeline
|
| 1958 |
-
pipeline = InferencePipeline(config)
|
| 1959 |
-
pipeline.initialize()
|
| 1960 |
-
|
| 1961 |
-
# Initialize video ingestion
|
| 1962 |
-
from ingestion import VideoIngestionService
|
| 1963 |
-
ingestion = VideoIngestionService(
|
| 1964 |
-
source=config['video_source'],
|
| 1965 |
-
frame_skip=config['frame_skip'],
|
| 1966 |
-
target_width=config['target_width'],
|
| 1967 |
-
queue_maxsize=30
|
| 1968 |
-
)
|
| 1969 |
-
|
| 1970 |
-
# Graceful shutdown handler
|
| 1971 |
-
shutdown_flag = False
|
| 1972 |
-
def signal_handler(sig, frame):
|
| 1973 |
-
nonlocal shutdown_flag
|
| 1974 |
-
logger.info("Shutdown signal received...")
|
| 1975 |
-
shutdown_flag = True
|
| 1976 |
-
signal.signal(signal.SIGINT, signal_handler)
|
| 1977 |
-
signal.signal(signal.SIGTERM, signal_handler)
|
| 1978 |
-
|
| 1979 |
-
# Initialize video writer for annotated output
|
| 1980 |
-
save_video = os.getenv('SAVE_ANNOTATED', 'true').lower() == 'true'
|
| 1981 |
-
video_writer = None
|
| 1982 |
-
output_path = '/data/output_annotated.mp4'
|
| 1983 |
-
|
| 1984 |
-
# Start processing
|
| 1985 |
-
ingestion.start()
|
| 1986 |
-
logger.info("Processing started. Press Ctrl+C to stop.")
|
| 1987 |
-
|
| 1988 |
-
fps_counter = 0
|
| 1989 |
-
fps_start = time.time()
|
| 1990 |
-
current_fps = -1.0 # Sentinel: -1 means "not yet computed"
|
| 1991 |
-
|
| 1992 |
-
while not shutdown_flag:
|
| 1993 |
-
packet = ingestion.get_frame(timeout=2.0)
|
| 1994 |
-
if packet is None:
|
| 1995 |
-
logger.info("End of video stream.")
|
| 1996 |
-
break
|
| 1997 |
-
|
| 1998 |
-
annotations = pipeline.process_frame(
|
| 1999 |
-
frame=packet.frame,
|
| 2000 |
-
frame_id=packet.frame_id,
|
| 2001 |
-
timestamp=packet.timestamp,
|
| 2002 |
-
source_id=packet.source_id,
|
| 2003 |
-
fps=getattr(ingestion, '_src_fps', 30.0)
|
| 2004 |
-
)
|
| 2005 |
-
|
| 2006 |
-
# Draw annotations and write to output video
|
| 2007 |
-
if save_video:
|
| 2008 |
-
if annotations:
|
| 2009 |
-
vis_frame = InferencePipeline.draw_annotations(
|
| 2010 |
-
packet.frame, annotations, packet.frame_id,
|
| 2011 |
-
timestamp=packet.timestamp, fps=current_fps
|
| 2012 |
-
)
|
| 2013 |
-
else:
|
| 2014 |
-
# Still write unannotated frames for smooth playback
|
| 2015 |
-
vis_frame = packet.frame.copy()
|
| 2016 |
-
mins = int(packet.timestamp // 60)
|
| 2017 |
-
secs = packet.timestamp % 60
|
| 2018 |
-
fps_str = f"{current_fps:.1f}" if current_fps >= 0 else "..."
|
| 2019 |
-
hud = f"{mins:02d}:{secs:05.2f} | Tracked: 0 | FPS: {fps_str}"
|
| 2020 |
-
cv2.putText(vis_frame, hud, (10, 25),
|
| 2021 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2, cv2.LINE_AA)
|
| 2022 |
-
if video_writer is None:
|
| 2023 |
-
h, w = vis_frame.shape[:2]
|
| 2024 |
-
src_fps = ingestion._src_fps if hasattr(ingestion, '_src_fps') else 30.0
|
| 2025 |
-
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
|
| 2026 |
-
video_writer = cv2.VideoWriter(output_path, fourcc, src_fps, (w, h))
|
| 2027 |
-
logger.info(f"Saving annotated video to {output_path} @ {src_fps} FPS")
|
| 2028 |
-
video_writer.write(vis_frame)
|
| 2029 |
-
|
| 2030 |
-
# FPS tracking
|
| 2031 |
-
fps_counter += 1
|
| 2032 |
-
elapsed = time.time() - fps_start
|
| 2033 |
-
# Update current_fps every frame so HUD never shows 0.0
|
| 2034 |
-
# Require at least 2 frames to avoid misleading spikes on first frame
|
| 2035 |
-
if elapsed > 0 and fps_counter >= 2:
|
| 2036 |
-
current_fps = fps_counter / elapsed
|
| 2037 |
-
if elapsed >= 5.0:
|
| 2038 |
-
logger.info(
|
| 2039 |
-
f"Pipeline FPS: {current_fps:.1f} | "
|
| 2040 |
-
f"Frames: {packet.frame_id} | "
|
| 2041 |
-
f"Dropped: {ingestion.frames_dropped}"
|
| 2042 |
-
)
|
| 2043 |
-
fps_counter = 0
|
| 2044 |
-
fps_start = time.time()
|
| 2045 |
-
|
| 2046 |
-
# Cleanup
|
| 2047 |
-
if video_writer is not None:
|
| 2048 |
-
video_writer.release()
|
| 2049 |
-
logger.info(f"Annotated video saved to {output_path}")
|
| 2050 |
-
ingestion.stop()
|
| 2051 |
-
pipeline.shutdown()
|
| 2052 |
-
logger.info("CV Inference service stopped.")
|
| 2053 |
|
|
|
|
| 2054 |
|
| 2055 |
-
|
| 2056 |
-
main()
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
tags:
|
| 6 |
+
- object-detection
|
| 7 |
+
- re-identification
|
| 8 |
+
- construction
|
| 9 |
+
- aerial-vision
|
| 10 |
+
- rf-detr
|
| 11 |
+
- dinov2
|
| 12 |
+
- osnet
|
| 13 |
+
- real-time
|
| 14 |
+
- tracking
|
| 15 |
+
pipeline_tag: object-detection
|
| 16 |
+
library_name: pytorch
|
| 17 |
+
datasets:
|
| 18 |
+
- roboflow
|
| 19 |
+
---
|
| 20 |
|
| 21 |
+
# 🏗️ SiteSense — Model Weights
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
+
**Real-Time Construction Equipment Monitoring via Aerial Computer Vision**
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
+
[](https://github.com/Mahmoud-Zaafan/asdfqer)
|
| 26 |
+
[](LICENSE)
|
| 27 |
+
[](https://python.org)
|
| 28 |
+
[](https://pytorch.org)
|
| 29 |
|
| 30 |
+
---
|
| 31 |
|
| 32 |
+
## Overview
|
|
|
|
| 33 |
|
| 34 |
+
This repository hosts the trained model weights for [SiteSense](https://github.com/Mahmoud-Zaafan/asdfqer) — a real-time pipeline that **detects, tracks, identifies, and classifies the activity** of heavy construction equipment from drone/aerial video footage.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
+
The system processes each frame through a multi-phase pipeline:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
+
```
|
| 39 |
+
Video Frame → RF-DETR Detection → BoT-SORT Tracking → DINOv3 Re-ID → Activity Classification → Kafka Events
|
| 40 |
+
```
|
| 41 |
|
| 42 |
+
---
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
+
## Model Weights
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
+
| File | Size | Architecture | Task | Training Data |
|
| 47 |
+
|:---|:---:|:---|:---|:---|
|
| 48 |
+
| `rfdetr_construction.pth` | 122 MB | RF-DETR (Real-time Foundation DETR) | 8-class object detection | Custom aerial construction dataset (Roboflow) |
|
| 49 |
+
| `dinov3_reid_head.pth` | 5.4 MB | Linear projection head (1536→256→128) | Equipment re-identification | Contrastive pairs from tracked equipment |
|
| 50 |
+
| `osnet_x0_25_msmt17.pt` | 2.9 MB | OSNet x0.25 | Appearance-based ReID for BoT-SORT | MSMT17 (pretrained) |
|
| 51 |
|
| 52 |
+
> **Note:** The DINOv3 ViT-B/16 backbone (~327 MB) is **not included** here. It is auto-downloaded from [facebook/dinov3-vitb16-pretrain-lvd1689m](https://huggingface.co/facebook/dinov3-vitb16-pretrain-lvd1689m) on first run using your `HF_TOKEN`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
+
---
|
|
|
|
|
|
|
| 55 |
|
| 56 |
+
## Detection Classes
|
|
|
|
| 57 |
|
| 58 |
+
The RF-DETR detector is fine-tuned to recognize **8 classes** of construction equipment from aerial perspectives:
|
| 59 |
|
| 60 |
+
| ID | Class | ID | Class |
|
| 61 |
+
|:---:|:---|:---:|:---|
|
| 62 |
+
| 0 | Excavator | 4 | Mobile Crane |
|
| 63 |
+
| 1 | Dump Truck | 5 | Tower Crane |
|
| 64 |
+
| 2 | Bulldozer | 6 | Roller Compactor |
|
| 65 |
+
| 3 | Wheel Loader | 7 | Cement Mixer |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
|
| 67 |
+
---
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
+
## Training Results
|
|
|
|
|
|
|
| 70 |
|
| 71 |
+
### RF-DETR Detector
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
+
| Metric | Value |
|
| 74 |
+
|:---|:---:|
|
| 75 |
+
| **mAP@50** | 0.8340 |
|
| 76 |
+
| **mAP@50:95** | 0.7607 |
|
| 77 |
+
| **F1 Score** | 0.8859 |
|
| 78 |
+
| **Precision** | 0.8666 |
|
| 79 |
+
| **Recall** | 0.9061 |
|
| 80 |
+
| Resolution | 560×560 |
|
| 81 |
+
| Epochs | 70 |
|
| 82 |
|
| 83 |
+
### DINOv3 Re-ID Projection Head
|
| 84 |
|
| 85 |
+
| Metric | Value |
|
| 86 |
+
|:---|:---:|
|
| 87 |
+
| **Contrastive Loss** | 0.0482 |
|
| 88 |
+
| **Accuracy** | 96.8% |
|
| 89 |
+
| Embedding Dim | 128-d L2-normalized |
|
| 90 |
+
| Training Pairs | ~12,000 positive pairs |
|
| 91 |
|
| 92 |
+
---
|
|
|
|
|
|
|
| 93 |
|
| 94 |
+
## Quick Start
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
|
| 96 |
+
### Option A: Download All Weights (Recommended)
|
|
|
|
|
|
|
| 97 |
|
| 98 |
+
```bash
|
| 99 |
+
pip install huggingface_hub
|
| 100 |
+
huggingface-cli download Zaafan/sitesense-weights --local-dir models/
|
| 101 |
+
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
+
### Option B: Python API
|
|
|
|
|
|
|
|
|
|
| 104 |
|
| 105 |
+
```python
|
| 106 |
+
from huggingface_hub import hf_hub_download
|
|
|
|
| 107 |
|
| 108 |
+
# Download individual weights
|
| 109 |
+
hf_hub_download(repo_id="Zaafan/sitesense-weights", filename="rfdetr_construction.pth", local_dir="models/")
|
| 110 |
+
hf_hub_download(repo_id="Zaafan/sitesense-weights", filename="dinov3_reid_head.pth", local_dir="models/")
|
| 111 |
+
hf_hub_download(repo_id="Zaafan/sitesense-weights", filename="osnet_x0_25_msmt17.pt", local_dir="models/")
|
| 112 |
+
```
|
| 113 |
|
| 114 |
+
### Option C: Auto-Download (Zero Setup)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
|
| 116 |
+
The SiteSense pipeline automatically downloads missing weights on first run:
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
+
```python
|
| 119 |
+
# In services/cv-inference/main.py — resolve_weights() handles this transparently
|
| 120 |
+
weights_path = resolve_weights('rfdetr_construction.pth') # local first, HF fallback
|
| 121 |
+
```
|
| 122 |
|
| 123 |
+
---
|
|
|
|
|
|
|
|
|
|
| 124 |
|
| 125 |
+
## Usage with SiteSense Pipeline
|
|
|
|
|
|
|
| 126 |
|
| 127 |
+
```bash
|
| 128 |
+
# 1. Clone the repository
|
| 129 |
+
git clone https://github.com/Mahmoud-Zaafan/asdfqer.git
|
| 130 |
+
cd asdfqer
|
|
|
|
|
|
|
| 131 |
|
| 132 |
+
# 2. Download weights
|
| 133 |
+
huggingface-cli download Zaafan/sitesense-weights --local-dir models/
|
| 134 |
|
| 135 |
+
# 3. Configure environment
|
| 136 |
+
cp .env.example .env
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
|
| 138 |
+
# 4. Launch infrastructure + run pipeline
|
| 139 |
+
docker compose up --build
|
| 140 |
+
docker compose --profile pipeline up cv-inference
|
| 141 |
+
```
|
| 142 |
|
| 143 |
+
---
|
|
|
|
| 144 |
|
| 145 |
+
## Citation
|
| 146 |
|
| 147 |
+
If you use these weights in your research or projects, please cite:
|
|
|
|
|
|
|
|
|
|
| 148 |
|
| 149 |
+
```bibtex
|
| 150 |
+
@misc{sitesense2025,
|
| 151 |
+
author = {Mahmoud Zaafan},
|
| 152 |
+
title = {SiteSense: Real-Time Construction Equipment Monitoring via Aerial Computer Vision},
|
| 153 |
+
year = {2025},
|
| 154 |
+
url = {https://github.com/Mahmoud-Zaafan/asdfqer}
|
| 155 |
+
}
|
| 156 |
+
```
|
| 157 |
|
| 158 |
+
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
|
| 160 |
+
## License
|
| 161 |
|
| 162 |
+
All weights are released under the [MIT License](LICENSE).
|
|
|