""" FLVE Engine - Face Liveness & Verification Engine. Provides: - Face detection (YOLO) - Face embedding extraction (ArcFace) - Liveness detection - Facial landmark processing (MediaPipe) """ import logging import time from pathlib import Path from typing import Optional, List, Dict, Any import cv2 import numpy as np import onnxruntime as ort from ..config import engine_config, cdn_config from ..models.schemas import BoundingBox, HeadPose, EyeState from .landmarks import get_landmark_processor logger = logging.getLogger(__name__) # Models directory MODELS_DIR = engine_config.models_dir def download_model(model_name: str) -> Path: """Download model from CDN if not exists""" model_path = MODELS_DIR / model_name if model_path.exists(): logger.info(f"Model already cached: {model_name}") return model_path # Create models directory MODELS_DIR.mkdir(parents=True, exist_ok=True) # Download from CDN cdn_url = f"{cdn_config.models_url}/{model_name}" logger.info(f"Downloading {model_name} from CDN: {cdn_url}") try: import requests # Try authenticated request if secret key provided if cdn_config.secret_key: try: import boto3 from botocore.client import Config s3_client = boto3.client( 's3', endpoint_url=cdn_config.endpoint, aws_access_key_id=cdn_config.access_key, aws_secret_access_key=cdn_config.secret_key, config=Config(signature_version='s3v4'), region_name=cdn_config.region ) # Models are stored at: labaidai/models/{model_name} s3_client.download_file( cdn_config.bucket, f'labaidai/models/{model_name}', str(model_path) ) size_mb = model_path.stat().st_size / (1024 * 1024) logger.info(f"Downloaded {model_name} via S3 ({size_mb:.2f} MB)") return model_path except Exception as e: logger.warning(f"S3 download failed, trying public URL: {e}") # Fallback to public URL response = requests.get(cdn_url, timeout=300) response.raise_for_status() with open(model_path, 'wb') as f: f.write(response.content) size_mb = model_path.stat().st_size / (1024 * 1024) logger.info(f"Downloaded {model_name} ({size_mb:.2f} MB)") return model_path except Exception as e: logger.error(f"Failed to download {model_name}: {e}") raise class FLVEEngine: """ Face Liveness & Verification Engine. Provides GPU-accelerated face processing pipeline: 1. Face detection (YOLO) 2. Face embedding extraction (ArcFace) 3. Liveness detection 4. Facial landmark processing (MediaPipe) """ def __init__(self): self.device = "cuda" if ort.get_device() == "GPU" else "cpu" logger.info(f"Initializing FLVE Engine on {self.device}") # ONNX Runtime session options sess_options = ort.SessionOptions() sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL providers = ( ['CUDAExecutionProvider', 'CPUExecutionProvider'] if self.device == "cuda" else ['CPUExecutionProvider'] ) # Load models self.detector = self._load_model("yolo-face.onnx", sess_options, providers) self.embedder = self._load_model("arcface.onnx", sess_options, providers) self.liveness, self.liveness_format, self.liveness_size = self._load_liveness_model( sess_options, providers ) # Landmark processor (lazy initialized) self._landmark_processor = None logger.info(f"FLVE Engine ready on {self.device}") @property def landmark_processor(self): """Lazy initialization of landmark processor""" if self._landmark_processor is None: try: self._landmark_processor = get_landmark_processor() except Exception as e: logger.warning(f"Landmark processor unavailable: {e}") return self._landmark_processor def _load_model( self, model_name: str, sess_options: ort.SessionOptions, providers: List[str] ) -> Optional[ort.InferenceSession]: """Load an ONNX model with automatic fallback to CPU.""" try: model_path = download_model(model_name) # Check file size size_mb = model_path.stat().st_size / (1024 * 1024) logger.info(f"Loading {model_name} ({size_mb:.2f} MB) with providers: {providers}") if size_mb < 0.1: logger.error(f"Model file too small ({size_mb:.2f} MB) - likely corrupt: {model_name}") return None # Try with requested providers first try: session = ort.InferenceSession(str(model_path), sess_options, providers=providers) logger.info(f"✓ Loaded: {model_name} with {providers}") return session except Exception as cuda_err: # If CUDA failed and we were trying CUDA, fall back to CPU only if 'CUDAExecutionProvider' in providers: logger.warning(f"CUDA failed for {model_name}: {cuda_err}") logger.info(f"Falling back to CPU for {model_name}") cpu_providers = ['CPUExecutionProvider'] session = ort.InferenceSession(str(model_path), sess_options, providers=cpu_providers) logger.info(f"✓ Loaded: {model_name} with CPU fallback") self.device = "cpu" # Update device status return session else: raise except Exception as e: logger.error(f"Failed to load {model_name}: {e}") import traceback traceback.print_exc() return None def _load_liveness_model( self, sess_options: ort.SessionOptions, providers: List[str] ) -> tuple: """Load liveness model and determine its input format.""" try: model_path = download_model("liveness.onnx") size_mb = model_path.stat().st_size / (1024 * 1024) logger.info(f"Loading liveness.onnx ({size_mb:.2f} MB) with providers: {providers}") if size_mb < 0.1: logger.error(f"Liveness model file too small ({size_mb:.2f} MB) - likely corrupt") return None, "channels_first", (128, 128) # Try with requested providers first, fallback to CPU try: session = ort.InferenceSession(str(model_path), sess_options, providers=providers) except Exception as cuda_err: if 'CUDAExecutionProvider' in providers: logger.warning(f"CUDA failed for liveness.onnx: {cuda_err}") logger.info("Falling back to CPU for liveness.onnx") session = ort.InferenceSession(str(model_path), sess_options, providers=['CPUExecutionProvider']) self.device = "cpu" else: raise input_shape = session.get_inputs()[0].shape logger.info(f"Loaded: liveness.onnx with input shape: {input_shape}") # Determine preprocessing based on shape if len(input_shape) == 4: if input_shape[1] == 3: # [batch, 3, H, W] - channels first liveness_format = "channels_first" liveness_size = (input_shape[2], input_shape[3]) elif input_shape[3] == 3: # [batch, H, W, 3] - channels last liveness_format = "channels_last" liveness_size = (input_shape[1], input_shape[2]) else: liveness_format = "channels_first" liveness_size = (128, 128) else: liveness_format = "channels_first" liveness_size = (128, 128) logger.info(f"Liveness preprocessing: {liveness_format}, size: {liveness_size}") return session, liveness_format, liveness_size except Exception as e: logger.error(f"Failed to load liveness model: {e}") import traceback traceback.print_exc() return None, "channels_first", (128, 128) def detect_faces(self, image: np.ndarray) -> List[Dict]: """Run YOLO face detection.""" if self.detector is None: return [] h, w = image.shape[:2] input_size = engine_config.yolo_input_size scale = min(input_size / w, input_size / h) new_w, new_h = int(w * scale), int(h * scale) # Preprocess resized = cv2.resize(image, (new_w, new_h)) padded = np.zeros((input_size, input_size, 3), dtype=np.uint8) padded[:new_h, :new_w] = resized blob = padded.astype(np.float32) / 255.0 blob = blob.transpose(2, 0, 1)[np.newaxis, ...] # Inference input_name = self.detector.get_inputs()[0].name outputs = self.detector.run(None, {input_name: blob}) return self._parse_yolo_output(outputs[0], w, h, scale) def _parse_yolo_output( self, output: np.ndarray, orig_w: int, orig_h: int, scale: float ) -> List[Dict]: """Parse YOLOv8 face detection output.""" faces = [] if output.shape[1] == 5: output = output[0].T else: output = output[0] for det in output: if len(det) >= 5: cx, cy, bw, bh, score = det[:5] if score < engine_config.detection_threshold: continue # Convert to original coordinates x1 = (cx - bw / 2) / scale y1 = (cy - bh / 2) / scale x2 = (cx + bw / 2) / scale y2 = (cy + bh / 2) / scale # Clamp x1 = max(0, min(orig_w, x1)) y1 = max(0, min(orig_h, y1)) x2 = max(0, min(orig_w, x2)) y2 = max(0, min(orig_h, y2)) # Extract 5-point landmarks if available landmarks = None if len(det) >= 15: landmarks = np.array([ [det[5] / scale, det[6] / scale], [det[7] / scale, det[8] / scale], [det[9] / scale, det[10] / scale], [det[11] / scale, det[12] / scale], [det[13] / scale, det[14] / scale], ]) faces.append({ 'box': [int(x1), int(y1), int(x2 - x1), int(y2 - y1)], 'confidence': float(score), 'landmarks': landmarks }) return self._nms(faces, engine_config.nms_threshold) def _nms(self, faces: List[Dict], threshold: float) -> List[Dict]: """Non-maximum suppression.""" if not faces: return [] faces = sorted(faces, key=lambda x: x['confidence'], reverse=True) keep = [] while faces: best = faces.pop(0) keep.append(best) faces = [f for f in faces if self._iou(best['box'], f['box']) < threshold] return keep def _iou(self, box1: List[int], box2: List[int]) -> float: """Calculate IoU between two boxes [x, y, w, h].""" x1, y1, w1, h1 = box1 x2, y2, w2, h2 = box2 xi1 = max(x1, x2) yi1 = max(y1, y2) xi2 = min(x1 + w1, x2 + w2) yi2 = min(y1 + h1, y2 + h2) inter = max(0, xi2 - xi1) * max(0, yi2 - yi1) union = w1 * h1 + w2 * h2 - inter return inter / union if union > 0 else 0 def extract_embedding(self, image: np.ndarray, box: List[int]) -> List[float]: """Extract 512-dim face embedding using ArcFace.""" if self.embedder is None: return [0.0] * 512 x, y, w, h = box # Crop with margin margin = 0.2 cx, cy = x + w / 2, y + h / 2 side = max(w, h) * (1 + margin * 2) x1 = int(max(0, cx - side / 2)) y1 = int(max(0, cy - side / 2)) x2 = int(min(image.shape[1], cx + side / 2)) y2 = int(min(image.shape[0], cy + side / 2)) face_crop = image[y1:y2, x1:x2] if face_crop.size == 0: return [0.0] * 512 # Resize to 112x112 face_crop = cv2.resize(face_crop, (engine_config.arcface_input_size, engine_config.arcface_input_size)) # Normalize input_shape = self.embedder.get_inputs()[0].shape input_name = self.embedder.get_inputs()[0].name blob = (face_crop.astype(np.float32) - 127.5) / 128.0 if input_shape[1] == 3: # NCHW blob = blob.transpose(2, 0, 1)[np.newaxis, ...] else: # NHWC blob = blob[np.newaxis, ...] outputs = self.embedder.run(None, {input_name: blob}) embedding = outputs[0][0] # L2 normalize norm = np.linalg.norm(embedding) if norm > 0: embedding = embedding / norm return embedding.tolist() def compute_liveness(self, image: np.ndarray, box: List[int]) -> float: """Compute liveness score.""" if self.liveness is None: return 0.5 x, y, w, h = box # Crop with larger margin for liveness margin = 0.35 cx, cy = x + w / 2, y + h / 2 side = max(w, h) * (1 + margin * 2) x1 = int(max(0, cx - side / 2)) y1 = int(max(0, cy - side / 2)) x2 = int(min(image.shape[1], cx + side / 2)) y2 = int(min(image.shape[0], cy + side / 2)) face_crop = image[y1:y2, x1:x2] if face_crop.size == 0: return 0.5 # Resize face_crop = cv2.resize(face_crop, self.liveness_size) face_crop = cv2.cvtColor(face_crop, cv2.COLOR_BGR2RGB) # Normalize (0-1) blob = face_crop.astype(np.float32) / 255.0 if self.liveness_format == "channels_first": blob = blob.transpose(2, 0, 1)[np.newaxis, ...].astype(np.float32) else: blob = blob[np.newaxis, ...].astype(np.float32) # Inference input_name = self.liveness.get_inputs()[0].name outputs = self.liveness.run(None, {input_name: blob}) logits = outputs[0][0] # Softmax exp_logits = np.exp(logits - np.max(logits)) probs = exp_logits / exp_logits.sum() # Index 0 is "real" p_real = float(probs[0]) logger.debug(f"Liveness score: {p_real:.4f}") return p_real def process(self, image: np.ndarray) -> Dict[str, Any]: """ Full inference pipeline. Returns dict with: - faces: List of face results - processing_ms: Processing time - device: Compute device - error: Error message if any """ start_time = time.time() faces = self.detect_faces(image) if not faces: return { 'faces': [], 'processing_ms': (time.time() - start_time) * 1000, 'device': self.device, 'error': 'No face detected' } results = [] for face in faces: box = face['box'] # Extract embedding embedding = self.extract_embedding(image, box) # Compute liveness liveness_score = self.compute_liveness(image, box) result = { 'box': BoundingBox(x=box[0], y=box[1], width=box[2], height=box[3]), 'confidence': face['confidence'], 'embedding': embedding, 'liveness_score': liveness_score, } # Process landmarks if available if self.landmark_processor: try: landmark_result = self.landmark_processor.process(image, box) if landmark_result.landmarks: result['landmarks'] = landmark_result.landmarks if landmark_result.eye_contours: result['eye_contours'] = landmark_result.eye_contours if landmark_result.head_pose: result['head_pose'] = HeadPose( yaw=landmark_result.head_pose.yaw, pitch=landmark_result.head_pose.pitch, roll=landmark_result.head_pose.roll ) if landmark_result.eye_state: result['eye_state'] = EyeState( left_openness=landmark_result.eye_state.left_openness, right_openness=landmark_result.eye_state.right_openness, is_blinking=landmark_result.eye_state.is_blinking, left_ear=landmark_result.eye_state.left_ear, right_ear=landmark_result.eye_state.right_ear, avg_ear=landmark_result.eye_state.avg_ear ) except Exception as e: logger.warning(f"Landmark processing failed: {e}") results.append(result) return { 'faces': results, 'processing_ms': (time.time() - start_time) * 1000, 'device': self.device, 'error': None } # Global engine instance _engine: Optional[FLVEEngine] = None startup_error: Optional[str] = None def get_engine() -> FLVEEngine: """Get the global FLVE engine instance.""" global _engine if _engine is None: raise RuntimeError("Engine not initialized. Call initialize_engine() first.") return _engine def initialize_engine() -> Optional[str]: """Initialize the global engine, return error message if failed.""" global _engine, startup_error try: _engine = FLVEEngine() startup_error = None return None except Exception as e: logger.error(f"Failed to initialize FLVE Engine: {e}") import traceback traceback.print_exc() startup_error = str(e) return str(e)