Spaces:
Sleeping
Sleeping
Download src/core/engine.py from farukhannan/flve: direct link, hf CLI and curl.
- Browser
- Download file 20 kB
-
https://huggingface.co/spaces/farukhannan/flve/resolve/main/src/core/engine.py
- Command line
-
hf download hf://spaces/farukhannan/flve/src/core/engine.py
-
curl -L -o engine.py https://huggingface.co/spaces/farukhannan/flve/resolve/main/src/core/engine.py
20 kB
| """ | |
| FLVE Engine - Face Liveness & Verification Engine. | |
| Provides: | |
| - Face detection (YOLO) | |
| - Face embedding extraction (ArcFace) | |
| - Liveness detection | |
| - Facial landmark processing (MediaPipe) | |
| """ | |
| import logging | |
| import time | |
| from pathlib import Path | |
| from typing import Optional, List, Dict, Any | |
| import cv2 | |
| import numpy as np | |
| import onnxruntime as ort | |
| from ..config import engine_config, cdn_config | |
| from ..models.schemas import BoundingBox, HeadPose, EyeState | |
| from .landmarks import get_landmark_processor | |
| logger = logging.getLogger(__name__) | |
| # Models directory | |
| MODELS_DIR = engine_config.models_dir | |
| def download_model(model_name: str) -> Path: | |
| """Download model from CDN if not exists""" | |
| model_path = MODELS_DIR / model_name | |
| if model_path.exists(): | |
| logger.info(f"Model already cached: {model_name}") | |
| return model_path | |
| # Create models directory | |
| MODELS_DIR.mkdir(parents=True, exist_ok=True) | |
| # Download from CDN | |
| cdn_url = f"{cdn_config.models_url}/{model_name}" | |
| logger.info(f"Downloading {model_name} from CDN: {cdn_url}") | |
| try: | |
| import requests | |
| # Try authenticated request if secret key provided | |
| if cdn_config.secret_key: | |
| try: | |
| import boto3 | |
| from botocore.client import Config | |
| s3_client = boto3.client( | |
| 's3', | |
| endpoint_url=cdn_config.endpoint, | |
| aws_access_key_id=cdn_config.access_key, | |
| aws_secret_access_key=cdn_config.secret_key, | |
| config=Config(signature_version='s3v4'), | |
| region_name=cdn_config.region | |
| ) | |
| # Models are stored at: labaidai/models/{model_name} | |
| s3_client.download_file( | |
| cdn_config.bucket, | |
| f'labaidai/models/{model_name}', | |
| str(model_path) | |
| ) | |
| size_mb = model_path.stat().st_size / (1024 * 1024) | |
| logger.info(f"Downloaded {model_name} via S3 ({size_mb:.2f} MB)") | |
| return model_path | |
| except Exception as e: | |
| logger.warning(f"S3 download failed, trying public URL: {e}") | |
| # Fallback to public URL | |
| response = requests.get(cdn_url, timeout=300) | |
| response.raise_for_status() | |
| with open(model_path, 'wb') as f: | |
| f.write(response.content) | |
| size_mb = model_path.stat().st_size / (1024 * 1024) | |
| logger.info(f"Downloaded {model_name} ({size_mb:.2f} MB)") | |
| return model_path | |
| except Exception as e: | |
| logger.error(f"Failed to download {model_name}: {e}") | |
| raise | |
| class FLVEEngine: | |
| """ | |
| Face Liveness & Verification Engine. | |
| Provides GPU-accelerated face processing pipeline: | |
| 1. Face detection (YOLO) | |
| 2. Face embedding extraction (ArcFace) | |
| 3. Liveness detection | |
| 4. Facial landmark processing (MediaPipe) | |
| """ | |
| def __init__(self): | |
| self.device = "cuda" if ort.get_device() == "GPU" else "cpu" | |
| logger.info(f"Initializing FLVE Engine on {self.device}") | |
| # ONNX Runtime session options | |
| sess_options = ort.SessionOptions() | |
| sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL | |
| providers = ( | |
| ['CUDAExecutionProvider', 'CPUExecutionProvider'] | |
| if self.device == "cuda" | |
| else ['CPUExecutionProvider'] | |
| ) | |
| # Load models | |
| self.detector = self._load_model("yolo-face.onnx", sess_options, providers) | |
| self.embedder = self._load_model("arcface.onnx", sess_options, providers) | |
| self.liveness, self.liveness_format, self.liveness_size = self._load_liveness_model( | |
| sess_options, providers | |
| ) | |
| # Landmark processor (lazy initialized) | |
| self._landmark_processor = None | |
| logger.info(f"FLVE Engine ready on {self.device}") | |
| def landmark_processor(self): | |
| """Lazy initialization of landmark processor""" | |
| if self._landmark_processor is None: | |
| try: | |
| self._landmark_processor = get_landmark_processor() | |
| except Exception as e: | |
| logger.warning(f"Landmark processor unavailable: {e}") | |
| return self._landmark_processor | |
| def _load_model( | |
| self, | |
| model_name: str, | |
| sess_options: ort.SessionOptions, | |
| providers: List[str] | |
| ) -> Optional[ort.InferenceSession]: | |
| """Load an ONNX model with automatic fallback to CPU.""" | |
| try: | |
| model_path = download_model(model_name) | |
| # Check file size | |
| size_mb = model_path.stat().st_size / (1024 * 1024) | |
| logger.info(f"Loading {model_name} ({size_mb:.2f} MB) with providers: {providers}") | |
| if size_mb < 0.1: | |
| logger.error(f"Model file too small ({size_mb:.2f} MB) - likely corrupt: {model_name}") | |
| return None | |
| # Try with requested providers first | |
| try: | |
| session = ort.InferenceSession(str(model_path), sess_options, providers=providers) | |
| logger.info(f"✓ Loaded: {model_name} with {providers}") | |
| return session | |
| except Exception as cuda_err: | |
| # If CUDA failed and we were trying CUDA, fall back to CPU only | |
| if 'CUDAExecutionProvider' in providers: | |
| logger.warning(f"CUDA failed for {model_name}: {cuda_err}") | |
| logger.info(f"Falling back to CPU for {model_name}") | |
| cpu_providers = ['CPUExecutionProvider'] | |
| session = ort.InferenceSession(str(model_path), sess_options, providers=cpu_providers) | |
| logger.info(f"✓ Loaded: {model_name} with CPU fallback") | |
| self.device = "cpu" # Update device status | |
| return session | |
| else: | |
| raise | |
| except Exception as e: | |
| logger.error(f"Failed to load {model_name}: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| return None | |
| def _load_liveness_model( | |
| self, | |
| sess_options: ort.SessionOptions, | |
| providers: List[str] | |
| ) -> tuple: | |
| """Load liveness model and determine its input format.""" | |
| try: | |
| model_path = download_model("liveness.onnx") | |
| size_mb = model_path.stat().st_size / (1024 * 1024) | |
| logger.info(f"Loading liveness.onnx ({size_mb:.2f} MB) with providers: {providers}") | |
| if size_mb < 0.1: | |
| logger.error(f"Liveness model file too small ({size_mb:.2f} MB) - likely corrupt") | |
| return None, "channels_first", (128, 128) | |
| # Try with requested providers first, fallback to CPU | |
| try: | |
| session = ort.InferenceSession(str(model_path), sess_options, providers=providers) | |
| except Exception as cuda_err: | |
| if 'CUDAExecutionProvider' in providers: | |
| logger.warning(f"CUDA failed for liveness.onnx: {cuda_err}") | |
| logger.info("Falling back to CPU for liveness.onnx") | |
| session = ort.InferenceSession(str(model_path), sess_options, providers=['CPUExecutionProvider']) | |
| self.device = "cpu" | |
| else: | |
| raise | |
| input_shape = session.get_inputs()[0].shape | |
| logger.info(f"Loaded: liveness.onnx with input shape: {input_shape}") | |
| # Determine preprocessing based on shape | |
| if len(input_shape) == 4: | |
| if input_shape[1] == 3: # [batch, 3, H, W] - channels first | |
| liveness_format = "channels_first" | |
| liveness_size = (input_shape[2], input_shape[3]) | |
| elif input_shape[3] == 3: # [batch, H, W, 3] - channels last | |
| liveness_format = "channels_last" | |
| liveness_size = (input_shape[1], input_shape[2]) | |
| else: | |
| liveness_format = "channels_first" | |
| liveness_size = (128, 128) | |
| else: | |
| liveness_format = "channels_first" | |
| liveness_size = (128, 128) | |
| logger.info(f"Liveness preprocessing: {liveness_format}, size: {liveness_size}") | |
| return session, liveness_format, liveness_size | |
| except Exception as e: | |
| logger.error(f"Failed to load liveness model: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| return None, "channels_first", (128, 128) | |
| def detect_faces(self, image: np.ndarray) -> List[Dict]: | |
| """Run YOLO face detection.""" | |
| if self.detector is None: | |
| return [] | |
| h, w = image.shape[:2] | |
| input_size = engine_config.yolo_input_size | |
| scale = min(input_size / w, input_size / h) | |
| new_w, new_h = int(w * scale), int(h * scale) | |
| # Preprocess | |
| resized = cv2.resize(image, (new_w, new_h)) | |
| padded = np.zeros((input_size, input_size, 3), dtype=np.uint8) | |
| padded[:new_h, :new_w] = resized | |
| blob = padded.astype(np.float32) / 255.0 | |
| blob = blob.transpose(2, 0, 1)[np.newaxis, ...] | |
| # Inference | |
| input_name = self.detector.get_inputs()[0].name | |
| outputs = self.detector.run(None, {input_name: blob}) | |
| return self._parse_yolo_output(outputs[0], w, h, scale) | |
| def _parse_yolo_output( | |
| self, | |
| output: np.ndarray, | |
| orig_w: int, | |
| orig_h: int, | |
| scale: float | |
| ) -> List[Dict]: | |
| """Parse YOLOv8 face detection output.""" | |
| faces = [] | |
| if output.shape[1] == 5: | |
| output = output[0].T | |
| else: | |
| output = output[0] | |
| for det in output: | |
| if len(det) >= 5: | |
| cx, cy, bw, bh, score = det[:5] | |
| if score < engine_config.detection_threshold: | |
| continue | |
| # Convert to original coordinates | |
| x1 = (cx - bw / 2) / scale | |
| y1 = (cy - bh / 2) / scale | |
| x2 = (cx + bw / 2) / scale | |
| y2 = (cy + bh / 2) / scale | |
| # Clamp | |
| x1 = max(0, min(orig_w, x1)) | |
| y1 = max(0, min(orig_h, y1)) | |
| x2 = max(0, min(orig_w, x2)) | |
| y2 = max(0, min(orig_h, y2)) | |
| # Extract 5-point landmarks if available | |
| landmarks = None | |
| if len(det) >= 15: | |
| landmarks = np.array([ | |
| [det[5] / scale, det[6] / scale], | |
| [det[7] / scale, det[8] / scale], | |
| [det[9] / scale, det[10] / scale], | |
| [det[11] / scale, det[12] / scale], | |
| [det[13] / scale, det[14] / scale], | |
| ]) | |
| faces.append({ | |
| 'box': [int(x1), int(y1), int(x2 - x1), int(y2 - y1)], | |
| 'confidence': float(score), | |
| 'landmarks': landmarks | |
| }) | |
| return self._nms(faces, engine_config.nms_threshold) | |
| def _nms(self, faces: List[Dict], threshold: float) -> List[Dict]: | |
| """Non-maximum suppression.""" | |
| if not faces: | |
| return [] | |
| faces = sorted(faces, key=lambda x: x['confidence'], reverse=True) | |
| keep = [] | |
| while faces: | |
| best = faces.pop(0) | |
| keep.append(best) | |
| faces = [f for f in faces if self._iou(best['box'], f['box']) < threshold] | |
| return keep | |
| def _iou(self, box1: List[int], box2: List[int]) -> float: | |
| """Calculate IoU between two boxes [x, y, w, h].""" | |
| x1, y1, w1, h1 = box1 | |
| x2, y2, w2, h2 = box2 | |
| xi1 = max(x1, x2) | |
| yi1 = max(y1, y2) | |
| xi2 = min(x1 + w1, x2 + w2) | |
| yi2 = min(y1 + h1, y2 + h2) | |
| inter = max(0, xi2 - xi1) * max(0, yi2 - yi1) | |
| union = w1 * h1 + w2 * h2 - inter | |
| return inter / union if union > 0 else 0 | |
| def extract_embedding(self, image: np.ndarray, box: List[int]) -> List[float]: | |
| """Extract 512-dim face embedding using ArcFace.""" | |
| if self.embedder is None: | |
| return [0.0] * 512 | |
| x, y, w, h = box | |
| # Crop with margin | |
| margin = 0.2 | |
| cx, cy = x + w / 2, y + h / 2 | |
| side = max(w, h) * (1 + margin * 2) | |
| x1 = int(max(0, cx - side / 2)) | |
| y1 = int(max(0, cy - side / 2)) | |
| x2 = int(min(image.shape[1], cx + side / 2)) | |
| y2 = int(min(image.shape[0], cy + side / 2)) | |
| face_crop = image[y1:y2, x1:x2] | |
| if face_crop.size == 0: | |
| return [0.0] * 512 | |
| # Resize to 112x112 | |
| face_crop = cv2.resize(face_crop, (engine_config.arcface_input_size, engine_config.arcface_input_size)) | |
| # Normalize | |
| input_shape = self.embedder.get_inputs()[0].shape | |
| input_name = self.embedder.get_inputs()[0].name | |
| blob = (face_crop.astype(np.float32) - 127.5) / 128.0 | |
| if input_shape[1] == 3: # NCHW | |
| blob = blob.transpose(2, 0, 1)[np.newaxis, ...] | |
| else: # NHWC | |
| blob = blob[np.newaxis, ...] | |
| outputs = self.embedder.run(None, {input_name: blob}) | |
| embedding = outputs[0][0] | |
| # L2 normalize | |
| norm = np.linalg.norm(embedding) | |
| if norm > 0: | |
| embedding = embedding / norm | |
| return embedding.tolist() | |
| def compute_liveness(self, image: np.ndarray, box: List[int]) -> float: | |
| """Compute liveness score.""" | |
| if self.liveness is None: | |
| return 0.5 | |
| x, y, w, h = box | |
| # Crop with larger margin for liveness | |
| margin = 0.35 | |
| cx, cy = x + w / 2, y + h / 2 | |
| side = max(w, h) * (1 + margin * 2) | |
| x1 = int(max(0, cx - side / 2)) | |
| y1 = int(max(0, cy - side / 2)) | |
| x2 = int(min(image.shape[1], cx + side / 2)) | |
| y2 = int(min(image.shape[0], cy + side / 2)) | |
| face_crop = image[y1:y2, x1:x2] | |
| if face_crop.size == 0: | |
| return 0.5 | |
| # Resize | |
| face_crop = cv2.resize(face_crop, self.liveness_size) | |
| face_crop = cv2.cvtColor(face_crop, cv2.COLOR_BGR2RGB) | |
| # Normalize (0-1) | |
| blob = face_crop.astype(np.float32) / 255.0 | |
| if self.liveness_format == "channels_first": | |
| blob = blob.transpose(2, 0, 1)[np.newaxis, ...].astype(np.float32) | |
| else: | |
| blob = blob[np.newaxis, ...].astype(np.float32) | |
| # Inference | |
| input_name = self.liveness.get_inputs()[0].name | |
| outputs = self.liveness.run(None, {input_name: blob}) | |
| logits = outputs[0][0] | |
| # Softmax | |
| exp_logits = np.exp(logits - np.max(logits)) | |
| probs = exp_logits / exp_logits.sum() | |
| # Index 0 is "real" | |
| p_real = float(probs[0]) | |
| logger.debug(f"Liveness score: {p_real:.4f}") | |
| return p_real | |
| def process(self, image: np.ndarray) -> Dict[str, Any]: | |
| """ | |
| Full inference pipeline. | |
| Returns dict with: | |
| - faces: List of face results | |
| - processing_ms: Processing time | |
| - device: Compute device | |
| - error: Error message if any | |
| """ | |
| start_time = time.time() | |
| faces = self.detect_faces(image) | |
| if not faces: | |
| return { | |
| 'faces': [], | |
| 'processing_ms': (time.time() - start_time) * 1000, | |
| 'device': self.device, | |
| 'error': 'No face detected' | |
| } | |
| results = [] | |
| for face in faces: | |
| box = face['box'] | |
| # Extract embedding | |
| embedding = self.extract_embedding(image, box) | |
| # Compute liveness | |
| liveness_score = self.compute_liveness(image, box) | |
| result = { | |
| 'box': BoundingBox(x=box[0], y=box[1], width=box[2], height=box[3]), | |
| 'confidence': face['confidence'], | |
| 'embedding': embedding, | |
| 'liveness_score': liveness_score, | |
| } | |
| # Process landmarks if available | |
| if self.landmark_processor: | |
| try: | |
| landmark_result = self.landmark_processor.process(image, box) | |
| if landmark_result.landmarks: | |
| result['landmarks'] = landmark_result.landmarks | |
| if landmark_result.eye_contours: | |
| result['eye_contours'] = landmark_result.eye_contours | |
| if landmark_result.head_pose: | |
| result['head_pose'] = HeadPose( | |
| yaw=landmark_result.head_pose.yaw, | |
| pitch=landmark_result.head_pose.pitch, | |
| roll=landmark_result.head_pose.roll | |
| ) | |
| if landmark_result.eye_state: | |
| result['eye_state'] = EyeState( | |
| left_openness=landmark_result.eye_state.left_openness, | |
| right_openness=landmark_result.eye_state.right_openness, | |
| is_blinking=landmark_result.eye_state.is_blinking, | |
| left_ear=landmark_result.eye_state.left_ear, | |
| right_ear=landmark_result.eye_state.right_ear, | |
| avg_ear=landmark_result.eye_state.avg_ear | |
| ) | |
| except Exception as e: | |
| logger.warning(f"Landmark processing failed: {e}") | |
| results.append(result) | |
| return { | |
| 'faces': results, | |
| 'processing_ms': (time.time() - start_time) * 1000, | |
| 'device': self.device, | |
| 'error': None | |
| } | |
| # Global engine instance | |
| _engine: Optional[FLVEEngine] = None | |
| startup_error: Optional[str] = None | |
| def get_engine() -> FLVEEngine: | |
| """Get the global FLVE engine instance.""" | |
| global _engine | |
| if _engine is None: | |
| raise RuntimeError("Engine not initialized. Call initialize_engine() first.") | |
| return _engine | |
| def initialize_engine() -> Optional[str]: | |
| """Initialize the global engine, return error message if failed.""" | |
| global _engine, startup_error | |
| try: | |
| _engine = FLVEEngine() | |
| startup_error = None | |
| return None | |
| except Exception as e: | |
| logger.error(f"Failed to initialize FLVE Engine: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| startup_error = str(e) | |
| return str(e) | |