Spaces:
Running
Running
jalon 3
Browse files- jericho/emotion.py +405 -0
- jericho/movement.py +452 -0
jericho/emotion.py
ADDED
|
@@ -0,0 +1,405 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse # pour lire arg de ligne de commande
|
| 2 |
+
import json # lire et écrire des fichiers JSON
|
| 3 |
+
import math # fonctions mathématiques
|
| 4 |
+
import os #module pour creer dossiers ect
|
| 5 |
+
import subprocess #lancer des programmes externes
|
| 6 |
+
import sys # pour forcer arret programme
|
| 7 |
+
import urllib.request # télécharger des fichiers depuis internet
|
| 8 |
+
import cv2 # importe OpenCV, bibliothèque pour lire, modifier et écrire des vidéos/images
|
| 9 |
+
import mediapipe as mp # IA la détection du corps/visage
|
| 10 |
+
import numpy as np # calcul
|
| 11 |
+
from mediapipe.tasks import python as mp_python
|
| 12 |
+
from mediapipe.tasks.python import vision as mp_vision # Importe les outils de vision par ordinateur de MediaPipe
|
| 13 |
+
from mediapipe.tasks.python.vision import FaceLandmarkerOptions, FaceLandmarker # Importe spécifiquement le détecteur de visage
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
MODEL_URL = "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task" #
|
| 18 |
+
MODEL_PATH = "face_landmarker.task" # IA save sur le pc
|
| 19 |
+
|
| 20 |
+
def ensure_model(): # verifie si L'IA est sur le pc
|
| 21 |
+
if not os.path.exists(MODEL_PATH): #
|
| 22 |
+
print(f"Téléchargement MediaPipe -> {MODEL_PATH} ...")
|
| 23 |
+
urllib.request.urlretrieve(MODEL_URL, MODEL_PATH)
|
| 24 |
+
print(" Modèle téléchargé.")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# Numero des points du visage
|
| 28 |
+
LEFT_EYE = [362, 385, 387, 263, 373, 380]
|
| 29 |
+
RIGHT_EYE = [33, 160, 158, 133, 153, 144]
|
| 30 |
+
MOUTH_OUTER = [61, 291, 0, 17, 37, 267]
|
| 31 |
+
MOUTH_L, MOUTH_R = 61, 291
|
| 32 |
+
LEFT_PUP, RIGHT_PUP = 468, 473
|
| 33 |
+
|
| 34 |
+
LEFT_BROW = [336, 296, 334, 293, 300, 285, 295, 282, 283, 276]
|
| 35 |
+
RIGHT_BROW = [107, 66, 105, 63, 70, 55, 53, 52, 65, 46]
|
| 36 |
+
LEFT_BROW_INNER = 285
|
| 37 |
+
RIGHT_BROW_INNER = 55
|
| 38 |
+
|
| 39 |
+
EAR_THRESHOLD = 0.20 # valeur limite d'ouverture de l'oeil
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class EMAFilter: # classe pour lisser les mouvements afin d eviter les micromouvements
|
| 44 |
+
|
| 45 |
+
def __init__(self, alpha=0.5):
|
| 46 |
+
self.alpha = alpha
|
| 47 |
+
self.state = {} # dictionnaire memorisant valeur precedente
|
| 48 |
+
|
| 49 |
+
def __call__(self, key, value):
|
| 50 |
+
if key not in self.state:
|
| 51 |
+
self.state[key] = value
|
| 52 |
+
else:
|
| 53 |
+
self.state[key] = self.alpha * value + (1.0 - self.alpha) * self.state[key]
|
| 54 |
+
return self.state[key]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _pts(lms, indices, w, h): # convertit point en px
|
| 59 |
+
return np.array([[lms[i].x * w, lms[i].y * h] for i in indices])
|
| 60 |
+
|
| 61 |
+
def ear(lms, indices, w, h): # calcule ouverture des yeaux
|
| 62 |
+
p = _pts(lms, indices, w, h) # coordonnée de l'oeil
|
| 63 |
+
A = np.linalg.norm(p[1]-p[5]) # distance vert oeil gauche
|
| 64 |
+
B = np.linalg.norm(p[2]-p[4]) # " " " droit
|
| 65 |
+
C = np.linalg.norm(p[0]-p[3]) # largeur oeil
|
| 66 |
+
return (A+B) / (2.0 * C + 1e-6) #formul calculant ear
|
| 67 |
+
|
| 68 |
+
def mar(lms, indices, w, h): # calcule ouverture bouche
|
| 69 |
+
p = _pts(lms, indices, w, h) # coordonnées des points de la bouche
|
| 70 |
+
A = np.linalg.norm(p[1]-p[5]) # hauteur gauche bpuche
|
| 71 |
+
B = np.linalg.norm(p[2]-p[4]) # hauteur droite bouche
|
| 72 |
+
C = np.linalg.norm(p[0]-p[3]) # Largeur de la bouche
|
| 73 |
+
return (A+B) / (2.0 * C + 1e-6) # Formule du MAR (plus la valeur est haute, plus la bouche est ouverte)
|
| 74 |
+
|
| 75 |
+
def smile_index(lms, w, h): # calcul valeur sourire
|
| 76 |
+
ml = np.array([lms[MOUTH_L].x*w, lms[MOUTH_L].y*h]) # pos en px coin gauche
|
| 77 |
+
mr = np.array([lms[MOUTH_R].x*w, lms[MOUTH_R].y*h]) #pos en px coin droit
|
| 78 |
+
pl = np.array([lms[LEFT_PUP].x*w, lms[LEFT_PUP].y*h]) # pos pupille gauche
|
| 79 |
+
pr = np.array([lms[RIGHT_PUP].x*w, lms[RIGHT_PUP].y*h]) # pos px pupille droite
|
| 80 |
+
return np.linalg.norm(mr-ml) / (np.linalg.norm(pr-pl) + 1e-6) # calcul sourire
|
| 81 |
+
|
| 82 |
+
def head_pose_from_matrix(matrix): # orientation de tete
|
| 83 |
+
R = matrix[:3, :3]
|
| 84 |
+
sy = math.sqrt(R[0,0]**2 + R[1,0]**2)
|
| 85 |
+
singular = sy < 1e-6
|
| 86 |
+
|
| 87 |
+
if not singular: # Si la position de la tête est normale
|
| 88 |
+
pitch = math.degrees(math.atan2(R[2,1], R[2,2]))
|
| 89 |
+
yaw = math.degrees(math.atan2(-R[2,0], sy))
|
| 90 |
+
roll = math.degrees(math.atan2(R[1,0], R[0,0]))
|
| 91 |
+
else: # si tete exetrememnt tourné
|
| 92 |
+
pitch = math.degrees(math.atan2(-R[1,2], R[1,1]))
|
| 93 |
+
yaw = math.degrees(math.atan2(-R[2,0], sy))
|
| 94 |
+
roll = 0.0
|
| 95 |
+
|
| 96 |
+
return yaw, pitch, roll
|
| 97 |
+
|
| 98 |
+
def face_bbox(lms, w, h): # rectangle entourant tete
|
| 99 |
+
xs = [lm.x for lm in lms]; ys = [lm.y for lm in lms]
|
| 100 |
+
xmn, xmx, ymn, ymx = min(xs), max(xs), min(ys), max(ys)
|
| 101 |
+
return (xmn+xmx)/2, (ymn+ymx)/2, xmx-xmn, ymx-ymn, xmn, ymn, xmx, ymx
|
| 102 |
+
|
| 103 |
+
def brow_raise(lms, brow_indices, eye_indices, w, h, face_h): #mesure a quel point sourcil levé
|
| 104 |
+
brow_pts = _pts(lms, brow_indices, w, h) # point sourcil en px
|
| 105 |
+
eye_pts = _pts(lms, eye_indices, w, h) #point yeux en px
|
| 106 |
+
brow_y = brow_pts[:, 1].mean() #moyenne hauteur tt les points sourcils
|
| 107 |
+
eye_y = eye_pts[:, 1].mean() # moyenne hauteur tt les points de l eoil
|
| 108 |
+
dist = eye_y - brow_y # distance oeil sourcil
|
| 109 |
+
return dist / (face_h * h + 1e-6) # divise par hauteur de visage
|
| 110 |
+
|
| 111 |
+
def brow_furrow(lms, w, h): #froncement des sourcils
|
| 112 |
+
li = np.array([lms[LEFT_BROW_INNER].x*w, lms[LEFT_BROW_INNER].y*h]) # coin intérieur gauche
|
| 113 |
+
ri = np.array([lms[RIGHT_BROW_INNER].x*w, lms[RIGHT_BROW_INNER].y*h]) # coin intérieur droit
|
| 114 |
+
pl = np.array([lms[LEFT_PUP].x*w, lms[LEFT_PUP].y*h]) # pupille gauche
|
| 115 |
+
pr = np.array([lms[RIGHT_PUP].x*w, lms[RIGHT_PUP].y*h]) # pupille droite
|
| 116 |
+
return np.linalg.norm(ri - li) / (np.linalg.norm(pr - pl) + 1e-6) # distance entre sourcils divisée par écart des yeux
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def emotions_records(records): # fonction analyse la video pour trouver les émotions
|
| 121 |
+
|
| 122 |
+
valid_recs = [r["rec"] for r in records if r["rec"].get("face_detected", False)] # garde image avec un visage detecté
|
| 123 |
+
if not valid_recs:
|
| 124 |
+
return
|
| 125 |
+
|
| 126 |
+
def get_mean(key, func=None): # calcul moyenne d'un signal
|
| 127 |
+
if func: return float(np.mean([func(r) for r in valid_recs]))
|
| 128 |
+
return float(np.mean([r[key] for r in valid_recs]))
|
| 129 |
+
|
| 130 |
+
def get_std(key, func=None): # calcul ecart type signal
|
| 131 |
+
if func: return max(float(np.std([func(r) for r in valid_recs])), 1e-6)
|
| 132 |
+
return max(float(np.std([r[key] for r in valid_recs])), 1e-6)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
smile_m = get_mean("smile") # moyenne sourire
|
| 136 |
+
smile_s = get_std("smile") # ecart-type sourire
|
| 137 |
+
ear_m = get_mean(None, lambda r: (r["ear_left"] + r["ear_right"]) / 2) # moyenne ouverture yeux
|
| 138 |
+
ear_s = get_std(None, lambda r: (r["ear_left"] + r["ear_right"]) / 2) # ecart-type ouverture yeux
|
| 139 |
+
mar_m = get_mean("mar") # moyenne ouverture bouche
|
| 140 |
+
mar_s = get_std("mar") # ecart-type ouverture bouche
|
| 141 |
+
b_raise_m = get_mean(None, lambda r: (r["brow_raise_left"] + r["brow_raise_right"]) / 2) # moyenne huateur soucrils
|
| 142 |
+
b_raise_s = get_std(None, lambda r: (r["brow_raise_left"] + r["brow_raise_right"]) / 2) # ecart type h sourcils
|
| 143 |
+
b_furrow_m = get_mean("brow_furrow") # moyenne froncement sourcils
|
| 144 |
+
b_furrow_s = get_std("brow_furrow") # ecart type froncement sourcil
|
| 145 |
+
|
| 146 |
+
for item in records: # pour chaque image de la video
|
| 147 |
+
rec = item["rec"] # o recupere les données de l image
|
| 148 |
+
if not rec.get("face_detected", False):
|
| 149 |
+
rec["emotion"] = "NEUTRE" # si pas d image, emotion neutre
|
| 150 |
+
continue
|
| 151 |
+
|
| 152 |
+
smile = rec["smile"]
|
| 153 |
+
ear = (rec["ear_left"] + rec["ear_right"]) / 2
|
| 154 |
+
mar = rec["mar"]
|
| 155 |
+
b_raise = (rec["brow_raise_left"] + rec["brow_raise_right"]) / 2
|
| 156 |
+
b_furrow = rec["brow_furrow"]
|
| 157 |
+
|
| 158 |
+
# calcul les Z-scores
|
| 159 |
+
n_smile = (smile - smile_m) / smile_s
|
| 160 |
+
n_ear = (ear - ear_m) / ear_s
|
| 161 |
+
n_mar = (mar - mar_m) / mar_s
|
| 162 |
+
n_brow_raise = (b_raise - b_raise_m) / b_raise_s
|
| 163 |
+
n_brow_furrow = (b_furrow - b_furrow_m) / b_furrow_s
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
if n_brow_raise > 0.8 and n_ear > 0.8:
|
| 169 |
+
emotion = "SURPRISE"
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
elif n_brow_raise < -0.5 and n_smile < -0.5:
|
| 173 |
+
emotion = "COLERE"
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
elif n_brow_raise < -0.3 and n_ear < -0.5:
|
| 177 |
+
emotion = "TRISTESSE"
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
elif n_smile > 0.8:
|
| 181 |
+
emotion = "JOIE"
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
elif n_ear > 0.8:
|
| 185 |
+
emotion = "PEUR"
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
elif n_smile > 0.3:
|
| 189 |
+
emotion = "CONTENT"
|
| 190 |
+
|
| 191 |
+
else:
|
| 192 |
+
emotion = "NEUTRE"
|
| 193 |
+
|
| 194 |
+
rec["emotion"] = emotion
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def annotate_frame(frame, lms, sig, w, h): # fonction pour annoter la video
|
| 198 |
+
|
| 199 |
+
#coordonee boite du visage
|
| 200 |
+
x1,y1 = int(sig["bbox_xmin"]*w), int(sig["bbox_ymin"]*h)
|
| 201 |
+
x2,y2 = int(sig["bbox_xmax"]*w), int(sig["bbox_ymax"]*h)
|
| 202 |
+
|
| 203 |
+
cv2.rectangle(frame,(x1,y1),(x2,y2),(0,255,0),2) # rectangle visage
|
| 204 |
+
|
| 205 |
+
# affiche l'emotion au dessus de la boite
|
| 206 |
+
emotion = sig.get("emotion", "NEUTRE")
|
| 207 |
+
cv2.putText(frame, f"EMOTION: {emotion}", (x1, max(y1-15, 30)),
|
| 208 |
+
cv2.FONT_HERSHEY_DUPLEX, 1.0, (0, 255, 255), 2)
|
| 209 |
+
# dessine des points sur chaque marqueur
|
| 210 |
+
for idx in LEFT_EYE + RIGHT_EYE + MOUTH_OUTER:
|
| 211 |
+
cv2.circle(frame,(int(lms[idx].x*w), int(lms[idx].y*h)), 2, (255,0,255), -1)
|
| 212 |
+
|
| 213 |
+
for idx in [LEFT_PUP, RIGHT_PUP]:
|
| 214 |
+
cv2.circle(frame,(int(lms[idx].x*w), int(lms[idx].y*h)), 3, (0,200,255), -1)
|
| 215 |
+
|
| 216 |
+
for idx in LEFT_BROW:
|
| 217 |
+
cv2.circle(frame,(int(lms[idx].x*w), int(lms[idx].y*h)), 2, (0,255,255), -1)
|
| 218 |
+
for idx in RIGHT_BROW:
|
| 219 |
+
cv2.circle(frame,(int(lms[idx].x*w), int(lms[idx].y*h)), 2, (0,165,255), -1)
|
| 220 |
+
|
| 221 |
+
lines = [ # toutes les info
|
| 222 |
+
f"Yaw:{sig['yaw']:+.1f} Pitch:{sig['pitch']:+.1f} Roll:{sig['roll']:+.1f}", # orientation tete
|
| 223 |
+
f"EAR L:{sig['ear_left']:.3f} R:{sig['ear_right']:.3f}", # ouverture des yeux
|
| 224 |
+
f"MAR:{sig['mar']:.3f} Smile:{sig['smile']:.3f}", # ouverture de la bouche et le sourire
|
| 225 |
+
# f"BrowRaise L:{sig['brow_raise_left']:.3f} R:{sig['brow_raise_right']:.3f}", # hauteur des sourcils
|
| 226 |
+
# f"BrowFurrow:{sig['brow_furrow']:.3f}", # froncement
|
| 227 |
+
f"T:{sig['timestamp_s']:.2f}s", # Affiche le temps en secondes
|
| 228 |
+
]
|
| 229 |
+
for i, line in enumerate(lines): # boucle pour ecrire les lignes au dessus
|
| 230 |
+
cv2.putText(frame, line, (x1, y2+25+i*18),
|
| 231 |
+
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0,255,255), 1, cv2.LINE_AA) #
|
| 232 |
+
|
| 233 |
+
#fleche orientation tete
|
| 234 |
+
nose = (int(lms[1].x*w), int(lms[1].y*h))
|
| 235 |
+
tip = (int(nose[0] + 60 * math.sin(math.radians(sig["yaw"]))),
|
| 236 |
+
int(nose[1] + 60 * math.sin(math.radians(sig["pitch"]))))
|
| 237 |
+
cv2.arrowedLine(frame, nose, tip, (0,100,255), 2, tipLength=0.3)
|
| 238 |
+
|
| 239 |
+
return frame # retourne l image annoté
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def merge_audio(original_video, silent_video, output_path): # remet son de la video original
|
| 243 |
+
try:
|
| 244 |
+
subprocess.run( #
|
| 245 |
+
["ffmpeg", "-version"],
|
| 246 |
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True
|
| 247 |
+
)
|
| 248 |
+
except (subprocess.CalledProcessError, FileNotFoundError): # Si FFmpeg n'est pas installé ou plante
|
| 249 |
+
print("\n ffmpeg introuvable. téléchargé le")
|
| 250 |
+
return False
|
| 251 |
+
|
| 252 |
+
cmd = [ # combine son et video
|
| 253 |
+
"ffmpeg", "-y", "-i", silent_video, "-i", original_video,
|
| 254 |
+
"-c", "copy", "-map", "0:v:0", "-map", "1:a:0", "-shortest", output_path
|
| 255 |
+
]
|
| 256 |
+
result = subprocess.run(cmd, capture_output=True, text=True) # exécute la ligne de commande
|
| 257 |
+
return result.returncode == 0
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def process_video(video_path, output_dir):
|
| 261 |
+
os.makedirs(output_dir, exist_ok=True)
|
| 262 |
+
ensure_model() # verifie si IA telechargé
|
| 263 |
+
|
| 264 |
+
cap = cv2.VideoCapture(video_path) # zopen cv ouvre vudei
|
| 265 |
+
if not cap.isOpened(): #
|
| 266 |
+
sys.exit(f"impossible d'ouvrir : {video_path}")
|
| 267 |
+
|
| 268 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0 # nb d image par sec
|
| 269 |
+
W = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # largeur video en px
|
| 270 |
+
H = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # h en px
|
| 271 |
+
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # nb img vid
|
| 272 |
+
print(f"Video : {W}x{H} {fps:.1f} fps {total} frames") # info de la vid en console
|
| 273 |
+
|
| 274 |
+
base_opts = mp_python.BaseOptions(model_asset_path=MODEL_PATH) #
|
| 275 |
+
options = FaceLandmarkerOptions( # configure ia
|
| 276 |
+
base_options=base_opts, # Passe les options de base
|
| 277 |
+
running_mode=mp_vision.RunningMode.VIDEO,
|
| 278 |
+
num_faces=1, #un visage
|
| 279 |
+
min_face_detection_confidence=0.5, # Tolérance de détection du visage à 50%
|
| 280 |
+
min_face_presence_confidence=0.5, # Tolérance de maintien du visage à 50%
|
| 281 |
+
min_tracking_confidence=0.5, # Tolérance de suivi de mouvement à 50%
|
| 282 |
+
output_face_blendshapes=False,
|
| 283 |
+
output_facial_transformation_matrixes=True, # matrice pour suivi de l'orientation
|
| 284 |
+
)
|
| 285 |
+
landmarker = FaceLandmarker.create_from_options(options) # lance l ia
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
print("\n Extraction des signaux faciaux...")
|
| 289 |
+
records = [] # liste stock donnee de tte les images
|
| 290 |
+
frame_idx = 0
|
| 291 |
+
ema = EMAFilter(alpha=0.5)
|
| 292 |
+
|
| 293 |
+
while True:
|
| 294 |
+
ret, frame = cap.read() # lit l'image suivante.ret faux si video fini
|
| 295 |
+
if not ret:
|
| 296 |
+
break
|
| 297 |
+
|
| 298 |
+
ts_ms = int(frame_idx * 1000 / fps) #timestamp actuel en ms
|
| 299 |
+
ts_s = frame_idx / fps #timestamp en s
|
| 300 |
+
|
| 301 |
+
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # convertit les couleur pour faire le lien entre open cv et mediapipe
|
| 302 |
+
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) # transforme img pour mediapipe
|
| 303 |
+
result = landmarker.detect_for_video(mp_image, ts_ms) # Ia trouve visage sur l img
|
| 304 |
+
|
| 305 |
+
if result.face_landmarks and result.facial_transformation_matrixes:
|
| 306 |
+
lms = result.face_landmarks[0] # recup point du visage
|
| 307 |
+
matrix = result.facial_transformation_matrixes[0] # recupere matrice orientation du visage
|
| 308 |
+
|
| 309 |
+
cx_raw, cy_raw, bw_raw, bh_raw, xmn_raw, ymn_raw, xmx_raw, ymx_raw = face_bbox(lms, W, H)
|
| 310 |
+
yaw_raw, pitch_raw, roll_raw = head_pose_from_matrix(matrix) # orientation tete
|
| 311 |
+
el_raw = ear(lms, LEFT_EYE, W, H) # ouverture oeil gauche
|
| 312 |
+
er_raw = ear(lms, RIGHT_EYE, W, H) # ouverture oiel droit
|
| 313 |
+
m_raw = mar(lms, MOUTH_OUTER, W, H) #ouverture bouche
|
| 314 |
+
sm_raw = smile_index(lms, W, H) # largeur sourire
|
| 315 |
+
br_left_raw = brow_raise(lms, LEFT_BROW, LEFT_EYE, W, H, bh_raw) # hauteur sourcil G
|
| 316 |
+
br_right_raw = brow_raise(lms, RIGHT_BROW, RIGHT_EYE, W, H, bh_raw) # hauteur sourcill D
|
| 317 |
+
bf_raw = brow_furrow(lms, W, H) # froncement sourcil
|
| 318 |
+
|
| 319 |
+
rec = { # dico
|
| 320 |
+
"frame": frame_idx, # num img
|
| 321 |
+
"timestamp_s": round(ts_s, 4), #temps en s
|
| 322 |
+
"face_detected": True,
|
| 323 |
+
"face_cx": round(ema('cx', cx_raw), 4), # centre x
|
| 324 |
+
"face_cy": round(ema('cy', cy_raw), 4), # centre Y
|
| 325 |
+
"face_w": round(ema('bw', bw_raw), 4), # largeur visage
|
| 326 |
+
"face_h": round(ema('bh', bh_raw), 4), #hauteur visage
|
| 327 |
+
"bbox_xmin": round(ema('xmn', xmn_raw), 4), # bord gauche
|
| 328 |
+
"bbox_ymin": round(ema('ymn', ymn_raw), 4), # bord haut
|
| 329 |
+
"bbox_xmax": round(ema('xmx', xmx_raw), 4), # bord droit
|
| 330 |
+
"bbox_ymax": round(ema('ymx', ymx_raw), 4), #bord bas
|
| 331 |
+
"yaw": round(ema('yaw', yaw_raw), 2), # rota G & D
|
| 332 |
+
"pitch": round(ema('pitch', pitch_raw), 2), # inclinaison haut bas
|
| 333 |
+
"roll": round(ema('roll', roll_raw), 2), # penchement
|
| 334 |
+
"ear_left": round(ema('el', el_raw), 4), # oeil gauche
|
| 335 |
+
"ear_right": round(ema('er', er_raw), 4), # oeil droit
|
| 336 |
+
"mar": round(ema('m', m_raw), 4), # Bouche lissée
|
| 337 |
+
"smile": round(ema('sm', sm_raw), 4), # Sourire lissé
|
| 338 |
+
"brow_raise_left": round(ema('br_l', br_left_raw), 4), # Sourcil gauche lissé
|
| 339 |
+
"brow_raise_right": round(ema('br_r', br_right_raw), 4), # Sourcil droit lissé
|
| 340 |
+
"brow_furrow": round(ema('bf', bf_raw), 4), # Froncement lissé
|
| 341 |
+
}
|
| 342 |
+
records.append({"rec": rec, "lms": lms}) # Ajoute ce gros dictionnaire, et les points originaux 'lms', à la liste globale
|
| 343 |
+
else: # Si l'IA n'a pas trouvé de visage
|
| 344 |
+
records.append({ # Ajoute un petit dictionnaire disant "Face Not Detected"
|
| 345 |
+
"rec": {"frame": frame_idx, "timestamp_s": round(ts_s, 4), "face_detected": False},
|
| 346 |
+
"lms": None # Et pas de points
|
| 347 |
+
})
|
| 348 |
+
|
| 349 |
+
frame_idx += 1 # Passe à l'image suivante
|
| 350 |
+
if frame_idx % 100 == 0: # Toutes les 100 images...
|
| 351 |
+
print(f" Frame {frame_idx}/{total} ({100*frame_idx//total}%)") # ... Affiche la progression dans la console
|
| 352 |
+
|
| 353 |
+
cap.release() # Relâche le fichier vidéo pour libérer la mémoire de l'ordinateur
|
| 354 |
+
landmarker.close() # Ferme l'IA MediaPipe proprement
|
| 355 |
+
|
| 356 |
+
# Calcul des émotions globales
|
| 357 |
+
emotions_records(records) # Appelle la grosse fonction FACS pour déduire la ligne "emotion" de tous les dictionnaires d'un coup
|
| 358 |
+
|
| 359 |
+
# ======= PASSE 2 : DESSIN SUR LA VIDÉO =======
|
| 360 |
+
print("\n[Passe 2/2] Création de la vidéo avec affichage des émotions...") # Informe du début de la passe graphique
|
| 361 |
+
cap = cv2.VideoCapture(video_path) # ROUVRE la vidéo originale depuis le tout début
|
| 362 |
+
silent_path = os.path.join(output_dir, "_annotated_silent.mp4") # Prépare le nom d'un fichier vidéo temporaire (sans son)
|
| 363 |
+
final_path = os.path.join(output_dir, "output_annotated.mp4") # Prépare le nom du fichier vidéo final (avec son)
|
| 364 |
+
writer = cv2.VideoWriter(silent_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (W, H)) # Configure l'outil pour écrire la nouvelle vidéo image par image
|
| 365 |
+
|
| 366 |
+
idx = 0 # Compteur d'images remis à 0
|
| 367 |
+
while True: # Boucle infinie
|
| 368 |
+
ret, frame = cap.read() # Lit l'image originale
|
| 369 |
+
if not ret: break # Stop à la fin de la vidéo
|
| 370 |
+
|
| 371 |
+
if idx < len(records): # Sécurité pour ne pas dépasser la taille de notre liste de données
|
| 372 |
+
item = records[idx] # Récupère les données de l'image actuelle
|
| 373 |
+
if item["rec"].get("face_detected", False): # Si un visage était présent...
|
| 374 |
+
frame = annotate_frame(frame, item["lms"], item["rec"], W, H) # ... On dessine tout par-dessus
|
| 375 |
+
else: # Sinon...
|
| 376 |
+
cv2.putText(frame, "Visage non detecte", (20,40), # ... On écrit "Visage non détecté" en rouge
|
| 377 |
+
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0,0,255), 2)
|
| 378 |
+
|
| 379 |
+
writer.write(frame) # Enregistre l'image modifiée dans la nouvelle vidéo
|
| 380 |
+
idx += 1 # Passe à la suivante
|
| 381 |
+
|
| 382 |
+
cap.release() # Relâche la vidéo originale
|
| 383 |
+
writer.release() # Clôture le fichier vidéo annoté (fini d'écrire)
|
| 384 |
+
|
| 385 |
+
# ── Sauvegarde ─────────────────────────────────────────────────────────────
|
| 386 |
+
if merge_audio(video_path, silent_path, final_path): # Tente de recoller le son original via FFmpeg
|
| 387 |
+
os.remove(silent_path) # Si succès, on supprime la vidéo muette temporaire
|
| 388 |
+
else: # Si échec (FFmpeg absent)
|
| 389 |
+
os.replace(silent_path, final_path) # On renomme la vidéo muette en fichier final
|
| 390 |
+
|
| 391 |
+
json_path = os.path.join(output_dir, "signals.json") # Prépare le chemin pour sauvegarder le JSON (signals.json)
|
| 392 |
+
with open(json_path, "w", encoding="utf-8") as f: # Ouvre le fichier en mode écriture ("w")
|
| 393 |
+
json_records = [item["rec"] for item in records] # Extrait uniquement les dicos "rec" (sans les 478 points 'lms' pour éviter un fichier trop lourd)
|
| 394 |
+
json.dump( # Exporte le tout en texte formaté JSON
|
| 395 |
+
{"video": os.path.abspath(video_path), "fps": round(fps, 4), "width": W, "height": H, "total_frames": total, "frames": json_records},
|
| 396 |
+
f, ensure_ascii=False, indent=2 # Avec une jolie indentation (2 espaces) pour que ça soit lisible
|
| 397 |
+
)
|
| 398 |
+
print(f"\n fait, fichiers dans : {os.path.abspath(output_dir)}/")
|
| 399 |
+
|
| 400 |
+
if __name__ == "__main__": # Cette ligne spéciale vérifie si on a lancé ce fichier directement depuis le terminal
|
| 401 |
+
parser = argparse.ArgumentParser() # initialise analyseur de ligne de commande
|
| 402 |
+
parser.add_argument("--video", required=True) # on doit fournir un chemin avec --video
|
| 403 |
+
parser.add_argument("--output_dir", default="output") #pour le odssier où on place les docs
|
| 404 |
+
args = parser.parse_args() # lit ce que l'utilisateur a tapé
|
| 405 |
+
process_video(args.video, args.output_dir) #
|
jericho/movement.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import time
|
| 3 |
+
import threading
|
| 4 |
+
import argparse
|
| 5 |
+
import sys
|
| 6 |
+
import math
|
| 7 |
+
import numpy as np
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
# Chargement signals.json en ne conservant que les frames avec un visage détecté
|
| 11 |
+
|
| 12 |
+
def load_signals(path: str) -> tuple[dict, list[dict]]:
|
| 13 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 14 |
+
data = json.load(f)
|
| 15 |
+
meta = {k: v for k, v in data.items() if k != "frames"}
|
| 16 |
+
frames = [fr for fr in data["frames"] if fr.get("face_detected", False)]
|
| 17 |
+
print(f"[Chargement] {len(frames)} frames avec visage sur "
|
| 18 |
+
f"{data.get('total_frames', '?')} total "
|
| 19 |
+
f"({data.get('fps', '?')} fps, durée ~{frames[-1]['timestamp_s']:.2f}s)")
|
| 20 |
+
return meta, frames
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
# filtre pour lisser les mouvements pour ne pas les avoir saccadé
|
| 24 |
+
|
| 25 |
+
def moving_average(values: list[float], window: int = 7) -> np.ndarray:
|
| 26 |
+
arr = np.asarray(values, dtype=float)
|
| 27 |
+
pad = window // 2
|
| 28 |
+
padded = np.pad(arr, pad_width=pad, mode="edge")
|
| 29 |
+
kernel = np.ones(window) / window
|
| 30 |
+
smoothed = np.convolve(padded, kernel, mode="valid")
|
| 31 |
+
return smoothed[: len(arr)]
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# Traduit les signaux faciaux en commandes pour le robot
|
| 35 |
+
|
| 36 |
+
class SignalMapper:
|
| 37 |
+
|
| 38 |
+
# Limites de mouvements du robot en rad/m
|
| 39 |
+
HEAD_YAW_MAX = np.deg2rad(5) # rotation horizontale tête : ±35°
|
| 40 |
+
HEAD_PITCH_MAX = np.deg2rad(15) # inclinaison haut/bas tête : ±25°
|
| 41 |
+
HEAD_ROLL_MAX = np.deg2rad(15) # inclinaison latérale tête : ±25°
|
| 42 |
+
HEAD_Y_MAX = 0.03 # déplacement latéral tête : ±3 cm
|
| 43 |
+
HEAD_Z_MAX = 0.03 # déplacement vertical tête : ±3 cm
|
| 44 |
+
BODY_YAW_MAX = np.deg2rad(15) # rotation corps : ±25°
|
| 45 |
+
ANTENNA_MAX = np.deg2rad(120) # débattement antennes : ±45°
|
| 46 |
+
ANTENNA_MAX_alt = np.deg2rad(45)
|
| 47 |
+
ANTENNA_MAX_col = np.deg2rad(110)
|
| 48 |
+
COMBINED_YAW_LIMIT = np.deg2rad(60) # contrainte combinée corps + tête en yaw
|
| 49 |
+
|
| 50 |
+
# Valeurs d'amplification des mouvements
|
| 51 |
+
YAW_AMP = 8.0
|
| 52 |
+
PITCH_AMP = 5.0
|
| 53 |
+
ROLL_AMP = 3.0
|
| 54 |
+
CX_TO_Y_SCALE = 0.12
|
| 55 |
+
CX_TO_BODY_YAW = 0.8
|
| 56 |
+
CY_TO_Z_SCALE = -0.10
|
| 57 |
+
|
| 58 |
+
SMOOTH_WINDOW_HEAD = 7 # Lissage pour la pose tête pour réduire le "bruit" (fluidifie) et avoir une meilleure réactivité
|
| 59 |
+
SMOOTH_WINDOW_EMOTION = 11 # lissage pour les émotions pour éviter les détections sur des clignements
|
| 60 |
+
|
| 61 |
+
def __init__(self, frames: list[dict]):
|
| 62 |
+
self.frames = frames
|
| 63 |
+
self._compute_baselines()
|
| 64 |
+
self.commands = self._build_commands()
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _compute_baselines(self):
|
| 68 |
+
|
| 69 |
+
# Moyenne
|
| 70 |
+
def mean(key, calc_func=None):
|
| 71 |
+
if calc_func: return float(np.mean([calc_func(f) for f in self.frames]))
|
| 72 |
+
return float(np.mean([f[key] for f in self.frames]))
|
| 73 |
+
|
| 74 |
+
# Ecart type à 0.000001 pour éviter la division par 0
|
| 75 |
+
def std(key, calc_func=None):
|
| 76 |
+
if calc_func: return max(float(np.std([calc_func(f) for f in self.frames])), 1e-6)
|
| 77 |
+
return max(float(np.std([f[key] for f in self.frames])), 1e-6)
|
| 78 |
+
|
| 79 |
+
self.yaw_mean = mean("yaw") # rotation gauche/droite moyenne (degrés)
|
| 80 |
+
self.pitch_mean = mean("pitch") # inclinaison haut/bas moyenne
|
| 81 |
+
self.roll_mean = mean("roll") # roulis moyen
|
| 82 |
+
self.cx_mean = mean("face_cx") # position X moyenne du visage dans le cadre
|
| 83 |
+
self.cy_mean = mean("face_cy") # position Y moyenne
|
| 84 |
+
|
| 85 |
+
# Etirement de la bouche
|
| 86 |
+
self.smile_mean = mean("smile")
|
| 87 |
+
self.smile_std = std("smile")
|
| 88 |
+
|
| 89 |
+
# Position des sourcils (hauteur) avec les deux sourcils au cas où un n'est pas détecter/visible
|
| 90 |
+
self.brow_raise_mean = mean(None, lambda f: (f["brow_raise_left"] + f["brow_raise_right"]) / 2)
|
| 91 |
+
self.brow_raise_std = std(None, lambda f: (f["brow_raise_left"] + f["brow_raise_right"]) / 2)
|
| 92 |
+
|
| 93 |
+
# Ecartement des sourcils
|
| 94 |
+
self.brow_furrow_mean = mean("brow_furrow")
|
| 95 |
+
self.brow_furrow_std = std("brow_furrow")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
self.ear_mean = mean(None, lambda f: (f["ear_left"] + f["ear_right"]) / 2)
|
| 99 |
+
self.ear_std = std(None, lambda f: (f["ear_left"] + f["ear_right"]) / 2)
|
| 100 |
+
|
| 101 |
+
# Même chose pour la bouche
|
| 102 |
+
self.mar_mean = mean("mar")
|
| 103 |
+
self.mar_std = std("mar")
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
# Définie l'émotion selon les valeurs en entrées
|
| 107 |
+
|
| 108 |
+
def _detect_emotion(self, smile, ear, mar, brow_raise, brow_furrow):
|
| 109 |
+
n_smile = (smile - self.smile_mean) / self.smile_std
|
| 110 |
+
n_ear = (ear - self.ear_mean) / self.ear_std
|
| 111 |
+
n_mar = (mar - self.mar_mean) / self.mar_std
|
| 112 |
+
n_brow_raise = (brow_raise - self.brow_raise_mean) / self.brow_raise_std
|
| 113 |
+
n_brow_furrow = (brow_furrow - self.brow_furrow_mean) / self.brow_furrow_std
|
| 114 |
+
|
| 115 |
+
# 1. SURPRISE : Écart yeux/sourcil élevé ET yeux grand ouverts
|
| 116 |
+
if n_brow_raise > 0.8 and n_ear > 0.8:
|
| 117 |
+
return "SURPRISE"
|
| 118 |
+
|
| 119 |
+
# 2. COLÈRE : Écart yeux/sourcil diminué ET largeur bouche diminuée
|
| 120 |
+
if n_brow_raise < -0.5 and n_smile < -0.5:
|
| 121 |
+
return "COLERE"
|
| 122 |
+
|
| 123 |
+
# 3. TRISTESSE : Écart yeux/sourcil réduit ET petite ouverture des yeux
|
| 124 |
+
if n_brow_raise < -0.3 and n_ear < -0.5:
|
| 125 |
+
return "TRISTESSE"
|
| 126 |
+
|
| 127 |
+
# 4. JOIE : Grand sourire
|
| 128 |
+
if n_smile > 0.8:
|
| 129 |
+
return "JOIE"
|
| 130 |
+
|
| 131 |
+
# 5. PEUR : Yeux écarquillés
|
| 132 |
+
if n_ear > 0.8:
|
| 133 |
+
return "PEUR"
|
| 134 |
+
|
| 135 |
+
# 6. CONTENT : Léger sourire
|
| 136 |
+
if n_smile > 0.3:
|
| 137 |
+
return "CONTENT"
|
| 138 |
+
|
| 139 |
+
return "NEUTRE"
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# Mouvements des antennes selon l'émotion détéctée
|
| 143 |
+
|
| 144 |
+
def _compute_antennas(self, emotion, t_state):
|
| 145 |
+
left_antenna, right_antenna = 0.0, 0.0
|
| 146 |
+
|
| 147 |
+
if emotion == "TRISTESSE":
|
| 148 |
+
target_angle = -1.0 * (1.0 - math.exp(-1.5 * t_state)) * self.ANTENNA_MAX
|
| 149 |
+
left_antenna = (target_angle - 0.2)
|
| 150 |
+
right_antenna = -(target_angle - 0.2)
|
| 151 |
+
|
| 152 |
+
elif emotion == "JOIE":
|
| 153 |
+
oscillation = 0.5 * math.sin(2 * math.pi * 4.0 * t_state)
|
| 154 |
+
angle = (0.5 + oscillation) * self.ANTENNA_MAX_alt
|
| 155 |
+
left_antenna = -(angle)
|
| 156 |
+
right_antenna = (angle)
|
| 157 |
+
|
| 158 |
+
elif emotion == "CONTENT":
|
| 159 |
+
oscillation = 0.8 * math.sin(2 * math.pi * 0.5 * t_state)
|
| 160 |
+
angle = oscillation * self.ANTENNA_MAX_alt
|
| 161 |
+
left_antenna, right_antenna = angle, angle
|
| 162 |
+
|
| 163 |
+
elif emotion == "SURPRISE":
|
| 164 |
+
angle = 1.0 * (1.0 - math.exp(-14.0 * t_state)) * self.ANTENNA_MAX_alt
|
| 165 |
+
left_antenna = -(angle-0.8)
|
| 166 |
+
right_antenna = (angle-0.8)
|
| 167 |
+
|
| 168 |
+
elif emotion == "NEUTRE":
|
| 169 |
+
angle = 1.0 * (1.0 - math.exp(-14.0 * t_state)) * self.ANTENNA_MAX_alt
|
| 170 |
+
left_antenna = -(angle-0.3)
|
| 171 |
+
right_antenna = (angle-0.3)
|
| 172 |
+
|
| 173 |
+
elif emotion == "COLERE":
|
| 174 |
+
if t_state < 0.15:
|
| 175 |
+
angle = -1.0 * (t_state / 0.15) * self.ANTENNA_MAX_col
|
| 176 |
+
elif t_state < 0.40:
|
| 177 |
+
angle = (-1.0 + 1.0 * ((t_state - 0.15) / 0.15)) * self.ANTENNA_MAX_col
|
| 178 |
+
elif t_state < 0.65:
|
| 179 |
+
angle = -1.0 * ((t_state - 0.3) / 0.15) * self.ANTENNA_MAX_col
|
| 180 |
+
else:
|
| 181 |
+
angle = -1.0 * self.ANTENNA_MAX_col
|
| 182 |
+
left_antenna = (angle)
|
| 183 |
+
right_antenna = -(angle)
|
| 184 |
+
|
| 185 |
+
elif emotion == "PEUR":
|
| 186 |
+
tremblement = 0.1 * math.sin(2 * math.pi * 15.0 * t_state)
|
| 187 |
+
angle = -1.0 *(1.0 - math.exp(-1.5 * t_state)+ tremblement) * self.ANTENNA_MAX
|
| 188 |
+
left_antenna = (angle - 0.2)
|
| 189 |
+
right_antenna = -(angle - 0.2)
|
| 190 |
+
|
| 191 |
+
return left_antenna, right_antenna
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
"""
|
| 195 |
+
1. Extraction des signaux bruts en tableaux
|
| 196 |
+
2. Lissage
|
| 197 |
+
3. Pour chaque frame :
|
| 198 |
+
a. Conversion des angles de tête (degrés en radians, amplification, clamp)
|
| 199 |
+
b. Calcul des translations de la tête (y, z) depuis la position du visage
|
| 200 |
+
c. Calcul du body_yaw avec contrainte combinée avec head_yaw
|
| 201 |
+
d. Détection de l'émotion
|
| 202 |
+
e. Mouvements des antennes
|
| 203 |
+
"""
|
| 204 |
+
|
| 205 |
+
def _build_commands(self) -> list[dict]:
|
| 206 |
+
n = len(self.frames)
|
| 207 |
+
|
| 208 |
+
raw_yaw = np.array([f["yaw"] - self.yaw_mean for f in self.frames])
|
| 209 |
+
raw_pitch = np.array([f["pitch"] - self.pitch_mean for f in self.frames])
|
| 210 |
+
raw_roll = np.array([f["roll"] - self.roll_mean for f in self.frames])
|
| 211 |
+
raw_cx = np.array([f["face_cx"] - self.cx_mean for f in self.frames])
|
| 212 |
+
raw_cy = np.array([f["face_cy"] - self.cy_mean for f in self.frames])
|
| 213 |
+
|
| 214 |
+
raw_smile = np.array([f["smile"] for f in self.frames])
|
| 215 |
+
raw_ear = np.array([(f["ear_left"] + f["ear_right"]) / 2 for f in self.frames])
|
| 216 |
+
raw_mar = np.array([f["mar"] for f in self.frames])
|
| 217 |
+
raw_brow_raise = np.array([(f["brow_raise_left"] + f["brow_raise_right"]) / 2 for f in self.frames])
|
| 218 |
+
raw_brow_furrow = np.array([f["brow_furrow"] for f in self.frames])
|
| 219 |
+
|
| 220 |
+
w_h = self.SMOOTH_WINDOW_HEAD
|
| 221 |
+
w_e = self.SMOOTH_WINDOW_EMOTION
|
| 222 |
+
|
| 223 |
+
s_yaw = moving_average(raw_yaw, w_h)
|
| 224 |
+
s_pitch = moving_average(raw_pitch, w_h)
|
| 225 |
+
s_roll = moving_average(raw_roll, w_h)
|
| 226 |
+
s_cx = moving_average(raw_cx, w_h)
|
| 227 |
+
s_cy = moving_average(raw_cy, w_h)
|
| 228 |
+
|
| 229 |
+
s_smile = moving_average(raw_smile, w_e)
|
| 230 |
+
s_ear = moving_average(raw_ear, w_e)
|
| 231 |
+
s_mar = moving_average(raw_mar, w_e)
|
| 232 |
+
s_brow_raise = moving_average(raw_brow_raise, w_e)
|
| 233 |
+
s_brow_furrow = moving_average(raw_brow_furrow, w_e)
|
| 234 |
+
|
| 235 |
+
commands = []
|
| 236 |
+
current_emotion = "NEUTRE"
|
| 237 |
+
emotion_start_time = 0.0
|
| 238 |
+
|
| 239 |
+
for i in range(n):
|
| 240 |
+
t = self.frames[i]["timestamp_s"]
|
| 241 |
+
|
| 242 |
+
head_yaw = self._clamp(np.deg2rad(s_yaw[i] * self.YAW_AMP), self.HEAD_YAW_MAX)
|
| 243 |
+
head_pitch = self._clamp(np.deg2rad(s_pitch[i] * self.PITCH_AMP), self.HEAD_PITCH_MAX)
|
| 244 |
+
head_roll = self._clamp(np.deg2rad(s_roll[i] * self.ROLL_AMP), self.HEAD_ROLL_MAX)
|
| 245 |
+
|
| 246 |
+
head_y = self._clamp(s_cx[i] * self.CX_TO_Y_SCALE, self.HEAD_Y_MAX)
|
| 247 |
+
head_z = self._clamp(s_cy[i] * self.CY_TO_Z_SCALE, self.HEAD_Z_MAX)
|
| 248 |
+
|
| 249 |
+
body_yaw = self._clamp(s_cx[i] * self.CX_TO_BODY_YAW, self.BODY_YAW_MAX)
|
| 250 |
+
|
| 251 |
+
diff = body_yaw - head_yaw
|
| 252 |
+
if abs(diff) > self.COMBINED_YAW_LIMIT:
|
| 253 |
+
body_yaw = head_yaw + np.sign(diff) * self.COMBINED_YAW_LIMIT
|
| 254 |
+
|
| 255 |
+
detected_emotion = self._detect_emotion(
|
| 256 |
+
s_smile[i], s_ear[i], s_mar[i], s_brow_raise[i], s_brow_furrow[i]
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
if detected_emotion != current_emotion:
|
| 260 |
+
current_emotion = detected_emotion
|
| 261 |
+
emotion_start_time = t
|
| 262 |
+
|
| 263 |
+
t_state = t - emotion_start_time
|
| 264 |
+
ant_left, ant_right = self._compute_antennas(current_emotion, t_state)
|
| 265 |
+
|
| 266 |
+
ant_left = self._clamp(ant_left, self.ANTENNA_MAX)
|
| 267 |
+
ant_right = self._clamp(ant_right, self.ANTENNA_MAX)
|
| 268 |
+
|
| 269 |
+
commands.append({
|
| 270 |
+
"timestamp_s": round(t, 4),
|
| 271 |
+
"head": {
|
| 272 |
+
"x": 0.0,
|
| 273 |
+
"y": round(float(head_y), 5),
|
| 274 |
+
"z": round(float(head_z), 5),
|
| 275 |
+
"roll": round(float(head_roll), 5),
|
| 276 |
+
"pitch": round(float(head_pitch), 5),
|
| 277 |
+
"yaw": round(float(head_yaw), 5),
|
| 278 |
+
},
|
| 279 |
+
"body_yaw": round(float(body_yaw), 5),
|
| 280 |
+
"antennas": [
|
| 281 |
+
round(float(ant_left), 5),
|
| 282 |
+
round(float(ant_right), 5),
|
| 283 |
+
],
|
| 284 |
+
"emotion": current_emotion
|
| 285 |
+
})
|
| 286 |
+
|
| 287 |
+
print(f"[Mapping] {len(commands)} commandes générées.")
|
| 288 |
+
return commands
|
| 289 |
+
|
| 290 |
+
@staticmethod
|
| 291 |
+
def _clamp(value: float, limit: float) -> float:
|
| 292 |
+
return float(max(-limit, min(limit, value)))
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
# Sauvegarde des mouvements générés
|
| 296 |
+
|
| 297 |
+
def save_movement(commands: list[dict], meta: dict, output_path: str):
|
| 298 |
+
movement = {
|
| 299 |
+
"source_video": meta.get("video", "unknown"),
|
| 300 |
+
"source_fps": meta.get("fps", 0),
|
| 301 |
+
"total_frames": len(commands),
|
| 302 |
+
"duration_s": commands[-1]["timestamp_s"] if commands else 0,
|
| 303 |
+
"frames": commands,
|
| 304 |
+
}
|
| 305 |
+
with open(output_path, "w", encoding="utf-8") as f:
|
| 306 |
+
json.dump(movement, f, indent=2)
|
| 307 |
+
print(f"[Sauvegarde] Mouvement enregistré dans '{output_path}' "
|
| 308 |
+
f"({len(commands)} frames, {movement['duration_s']:.2f}s)")
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
# Refait les mouvements pour le robot
|
| 312 |
+
|
| 313 |
+
class SignalReplayApp:
|
| 314 |
+
|
| 315 |
+
def __init__(self, commands: list[dict]):
|
| 316 |
+
self.commands = commands
|
| 317 |
+
|
| 318 |
+
def run(self, reachy_mini, stop_event: threading.Event):
|
| 319 |
+
from reachy_mini.utils import create_head_pose
|
| 320 |
+
|
| 321 |
+
if not self.commands:
|
| 322 |
+
print("[Rejeu] Aucune commande à rejouer.")
|
| 323 |
+
return
|
| 324 |
+
|
| 325 |
+
print("[Rejeu] Initialisation de la pose de départ...")
|
| 326 |
+
reachy_mini.goto_target(
|
| 327 |
+
create_head_pose(), antennas=[0.0, 0.0], duration=1.0
|
| 328 |
+
)
|
| 329 |
+
time.sleep(1.2)
|
| 330 |
+
|
| 331 |
+
print(f"[Rejeu] Début — {len(self.commands)} frames "
|
| 332 |
+
f"sur {self.commands[-1]['timestamp_s']:.2f}s")
|
| 333 |
+
|
| 334 |
+
start_wall = time.time()
|
| 335 |
+
video_offset = self.commands[0]["timestamp_s"]
|
| 336 |
+
idx = 0
|
| 337 |
+
n = len(self.commands)
|
| 338 |
+
|
| 339 |
+
while not stop_event.is_set() and idx < n:
|
| 340 |
+
elapsed = time.time() - start_wall
|
| 341 |
+
target_t = video_offset + elapsed
|
| 342 |
+
|
| 343 |
+
# avance les frames si il y a un retard
|
| 344 |
+
while idx < n - 1 and self.commands[idx]["timestamp_s"] < target_t:
|
| 345 |
+
idx += 1
|
| 346 |
+
|
| 347 |
+
if idx >= n:
|
| 348 |
+
break
|
| 349 |
+
|
| 350 |
+
cmd = self.commands[idx]
|
| 351 |
+
head_dict = cmd["head"]
|
| 352 |
+
|
| 353 |
+
head_pose = create_head_pose(
|
| 354 |
+
x=head_dict["x"],
|
| 355 |
+
y=head_dict["y"],
|
| 356 |
+
z=head_dict["z"],
|
| 357 |
+
roll=head_dict["roll"],
|
| 358 |
+
pitch=head_dict["pitch"],
|
| 359 |
+
yaw=head_dict["yaw"],
|
| 360 |
+
degrees=False,
|
| 361 |
+
mm=False,
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
try:
|
| 365 |
+
reachy_mini.set_target(
|
| 366 |
+
head=head_pose,
|
| 367 |
+
antennas=cmd["antennas"],
|
| 368 |
+
body_yaw=cmd["body_yaw"],
|
| 369 |
+
)
|
| 370 |
+
except TypeError:
|
| 371 |
+
reachy_mini.set_target(head=head_pose, antennas=cmd["antennas"])
|
| 372 |
+
|
| 373 |
+
time.sleep(0.01) # 100hz
|
| 374 |
+
|
| 375 |
+
print("[Rejeu] Terminé. Retour à la pose neutre.")
|
| 376 |
+
reachy_mini.goto_target(
|
| 377 |
+
create_head_pose(), antennas=[0.0, 0.0], duration=1.0
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
"""
|
| 382 |
+
python signal_replay.py # rejeu depuis signals.json
|
| 383 |
+
python signal_replay.py ma_video_signals.json # choix de la source
|
| 384 |
+
python signal_replay.py --save movement.json # sauvegarde + rejeu
|
| 385 |
+
python signal_replay.py --save out.json --no-replay # sauvegarde seule sans le rejeu
|
| 386 |
+
"""
|
| 387 |
+
|
| 388 |
+
def parse_args():
|
| 389 |
+
parser = argparse.ArgumentParser(
|
| 390 |
+
description="Rejoue les signaux faciaux sur Reachy Mini."
|
| 391 |
+
)
|
| 392 |
+
parser.add_argument(
|
| 393 |
+
"signals_path",
|
| 394 |
+
nargs="?",
|
| 395 |
+
default="signals.json",
|
| 396 |
+
help="Chemin vers signals.json (défaut: signals.json)",
|
| 397 |
+
)
|
| 398 |
+
parser.add_argument(
|
| 399 |
+
"--save",
|
| 400 |
+
metavar="OUTPUT",
|
| 401 |
+
default=None,
|
| 402 |
+
help="Sauvegarder le mouvement généré dans un fichier JSON",
|
| 403 |
+
)
|
| 404 |
+
parser.add_argument(
|
| 405 |
+
"--no-replay",
|
| 406 |
+
action="store_true",
|
| 407 |
+
help="Générer et sauvegarder sans lancer la simulation",
|
| 408 |
+
)
|
| 409 |
+
return parser.parse_args()
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
"""
|
| 413 |
+
1. Parse les arguments
|
| 414 |
+
2. Charge les signaux faciaux
|
| 415 |
+
3. Génère les commandes robot avec SignalMapper
|
| 416 |
+
---Selon les arguments---
|
| 417 |
+
4. Sauvegarde optionnelle
|
| 418 |
+
5. Rejeu sur le robot
|
| 419 |
+
"""
|
| 420 |
+
|
| 421 |
+
def main():
|
| 422 |
+
args = parse_args()
|
| 423 |
+
|
| 424 |
+
meta, frames = load_signals(args.signals_path)
|
| 425 |
+
mapper = SignalMapper(frames)
|
| 426 |
+
|
| 427 |
+
if args.save:
|
| 428 |
+
save_movement(mapper.commands, meta, args.save)
|
| 429 |
+
|
| 430 |
+
if args.no_replay:
|
| 431 |
+
print("[Mode --no-replay] Génération terminée, pas de simulation.")
|
| 432 |
+
return
|
| 433 |
+
|
| 434 |
+
try:
|
| 435 |
+
from reachy_mini import ReachyMini
|
| 436 |
+
except ImportError:
|
| 437 |
+
print("[Erreur] reachy_mini non disponible. Lance avec --no-replay pour "
|
| 438 |
+
"tester sans simulation.")
|
| 439 |
+
sys.exit(1)
|
| 440 |
+
|
| 441 |
+
app = SignalReplayApp(mapper.commands)
|
| 442 |
+
|
| 443 |
+
with ReachyMini(media_backend="no_media") as robot:
|
| 444 |
+
stop_event = threading.Event()
|
| 445 |
+
try:
|
| 446 |
+
app.run(robot, stop_event)
|
| 447 |
+
except KeyboardInterrupt:
|
| 448 |
+
print("\n[Interruption] Arrêt demandé par l'utilisateur.")
|
| 449 |
+
stop_event.set()
|
| 450 |
+
|
| 451 |
+
if __name__ == "__main__":
|
| 452 |
+
main()
|