Spaces:
Running
Running
Merge branch 'main' of https://huggingface.co/spaces/Jarilo-Project/jericho
Browse files- jericho/main.py +217 -1
- jericho/static/main.js +10 -2
jericho/main.py
CHANGED
|
@@ -256,9 +256,219 @@ class PlaybackPayload(BaseModel):
|
|
| 256 |
playing: bool | None = None
|
| 257 |
action: str = "sync"
|
| 258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
class Jericho(ReachyMiniApp):
|
| 260 |
custom_app_url: str | None = "http://0.0.0.0:8042"
|
| 261 |
-
request_media_backend: str | None =
|
| 262 |
|
| 263 |
def __init__(self, **kwargs):
|
| 264 |
super().__init__(**kwargs)
|
|
@@ -435,6 +645,7 @@ class Jericho(ReachyMiniApp):
|
|
| 435 |
if os.path.exists(project_path) and os.path.isdir(project_path):
|
| 436 |
try:
|
| 437 |
shutil.rmtree(project_path)
|
|
|
|
| 438 |
print(f"[{project_name}] Projet supprimé avec succès.")
|
| 439 |
return {"status": "success"}
|
| 440 |
except Exception as e:
|
|
@@ -612,6 +823,7 @@ class Jericho(ReachyMiniApp):
|
|
| 612 |
state = dict(self.playback_state)
|
| 613 |
|
| 614 |
if state.get("stop_requested"):
|
|
|
|
| 615 |
reachy_mini.goto_target(create_head_pose(), antennas=[0.0, 0.0], duration=1.0)
|
| 616 |
with self.playback_lock:
|
| 617 |
self.playback_state["stop_requested"] = False
|
|
@@ -658,6 +870,8 @@ class Jericho(ReachyMiniApp):
|
|
| 658 |
except TypeError:
|
| 659 |
reachy_mini.set_target(head=head_pose, antennas=antennas_target)
|
| 660 |
|
|
|
|
|
|
|
| 661 |
if dirty:
|
| 662 |
with self.playback_lock:
|
| 663 |
|
|
@@ -666,6 +880,8 @@ class Jericho(ReachyMiniApp):
|
|
| 666 |
|
| 667 |
time.sleep(0.01 if playing else 0.03)
|
| 668 |
|
|
|
|
|
|
|
| 669 |
if __name__ == "__main__":
|
| 670 |
app = Jericho()
|
| 671 |
try:
|
|
|
|
| 256 |
playing: bool | None = None
|
| 257 |
action: str = "sync"
|
| 258 |
|
| 259 |
+
class RobotAudioStreamer:
|
| 260 |
+
"""Stream l'audio de la vidéo vers le haut-parleur Reachy Mini.
|
| 261 |
+
|
| 262 |
+
Le navigateur garde la vidéo pour l'image uniquement. L'audio est extrait
|
| 263 |
+
depuis output_annotated.mp4, converti en float32 16 kHz stéréo, puis poussé
|
| 264 |
+
par petits blocs dans mini.media.push_audio_sample().
|
| 265 |
+
"""
|
| 266 |
+
|
| 267 |
+
SAMPLE_RATE = 16000
|
| 268 |
+
CHANNELS = 2
|
| 269 |
+
CHUNK_SECONDS = 0.08
|
| 270 |
+
QUEUE_AHEAD_SECONDS = 0.20
|
| 271 |
+
RESYNC_THRESHOLD_SECONDS = 0.25
|
| 272 |
+
|
| 273 |
+
def __init__(self, app: "Jericho"):
|
| 274 |
+
self.app = app
|
| 275 |
+
self.audio_cache: dict[str, np.ndarray] = {}
|
| 276 |
+
self.current_project: str | None = None
|
| 277 |
+
self.next_push_time = 0.0
|
| 278 |
+
self.output_started = False
|
| 279 |
+
self.last_state_signature = None
|
| 280 |
+
self.last_dirty_updated_at = None
|
| 281 |
+
self.last_playing = False
|
| 282 |
+
self.audio_available = True
|
| 283 |
+
|
| 284 |
+
def _project_video_path(self, project_name: str) -> str:
|
| 285 |
+
return os.path.join(self.app.projects_dir, project_name, "output_annotated.mp4")
|
| 286 |
+
|
| 287 |
+
def _load_audio(self, project_name: str) -> np.ndarray:
|
| 288 |
+
if project_name in self.audio_cache:
|
| 289 |
+
return self.audio_cache[project_name]
|
| 290 |
+
|
| 291 |
+
video_path = self._project_video_path(project_name)
|
| 292 |
+
if not os.path.exists(video_path):
|
| 293 |
+
self.audio_cache[project_name] = np.zeros((0, self.CHANNELS), dtype=np.float32)
|
| 294 |
+
return self.audio_cache[project_name]
|
| 295 |
+
|
| 296 |
+
cmd = [
|
| 297 |
+
"ffmpeg", "-v", "error",
|
| 298 |
+
"-i", video_path,
|
| 299 |
+
"-vn",
|
| 300 |
+
"-ac", str(self.CHANNELS),
|
| 301 |
+
"-ar", str(self.SAMPLE_RATE),
|
| 302 |
+
"-f", "f32le",
|
| 303 |
+
"pipe:1",
|
| 304 |
+
]
|
| 305 |
+
|
| 306 |
+
try:
|
| 307 |
+
process = subprocess.run(cmd, capture_output=True, check=True)
|
| 308 |
+
audio = np.frombuffer(process.stdout, dtype=np.float32)
|
| 309 |
+
if audio.size == 0:
|
| 310 |
+
samples = np.zeros((0, self.CHANNELS), dtype=np.float32)
|
| 311 |
+
else:
|
| 312 |
+
usable = audio.size - (audio.size % self.CHANNELS)
|
| 313 |
+
samples = audio[:usable].reshape(-1, self.CHANNELS).astype(np.float32, copy=False)
|
| 314 |
+
except Exception as exc:
|
| 315 |
+
print(f"[Audio robot] Audio indisponible pour {project_name}: {exc}")
|
| 316 |
+
samples = np.zeros((0, self.CHANNELS), dtype=np.float32)
|
| 317 |
+
|
| 318 |
+
self.audio_cache[project_name] = samples
|
| 319 |
+
return samples
|
| 320 |
+
|
| 321 |
+
def _media(self, reachy_mini: ReachyMini):
|
| 322 |
+
return getattr(reachy_mini, "media", None)
|
| 323 |
+
|
| 324 |
+
def _start_output(self, reachy_mini: ReachyMini):
|
| 325 |
+
if self.output_started or not self.audio_available:
|
| 326 |
+
return
|
| 327 |
+
|
| 328 |
+
media = self._media(reachy_mini)
|
| 329 |
+
if media is None:
|
| 330 |
+
self.audio_available = False
|
| 331 |
+
print("[Audio robot] Backend média non disponible : audio robot désactivé.")
|
| 332 |
+
return
|
| 333 |
+
|
| 334 |
+
try:
|
| 335 |
+
media.start_playing()
|
| 336 |
+
# Réduit l'avance audio en file d'attente quand le backend le supporte.
|
| 337 |
+
for obj in (media, getattr(media, "audio", None)):
|
| 338 |
+
if obj is not None and hasattr(obj, "set_max_output_buffers"):
|
| 339 |
+
obj.set_max_output_buffers(3)
|
| 340 |
+
self.output_started = True
|
| 341 |
+
except Exception as exc:
|
| 342 |
+
self.audio_available = False
|
| 343 |
+
print(f"[Audio robot] Impossible de démarrer la sortie audio : {exc}")
|
| 344 |
+
|
| 345 |
+
def _flush_output(self, reachy_mini: ReachyMini):
|
| 346 |
+
"""Vide le son déjà mis en queue pour éviter qu'il continue après pause/seek."""
|
| 347 |
+
media = self._media(reachy_mini)
|
| 348 |
+
if media is None or not self.output_started:
|
| 349 |
+
return
|
| 350 |
+
|
| 351 |
+
flushed = False
|
| 352 |
+
for obj in (media, getattr(media, "audio", None)):
|
| 353 |
+
if obj is None:
|
| 354 |
+
continue
|
| 355 |
+
for method_name in ("clear_player", "clear_output_buffer"):
|
| 356 |
+
method = getattr(obj, method_name, None)
|
| 357 |
+
if callable(method):
|
| 358 |
+
try:
|
| 359 |
+
method()
|
| 360 |
+
flushed = True
|
| 361 |
+
except Exception:
|
| 362 |
+
pass
|
| 363 |
+
|
| 364 |
+
if not flushed:
|
| 365 |
+
# Fallback générique : redémarrer la pipeline audio vide la plupart des backends.
|
| 366 |
+
try:
|
| 367 |
+
media.stop_playing()
|
| 368 |
+
media.start_playing()
|
| 369 |
+
except Exception:
|
| 370 |
+
pass
|
| 371 |
+
|
| 372 |
+
def _stop_output(self, reachy_mini: ReachyMini):
|
| 373 |
+
media = self._media(reachy_mini)
|
| 374 |
+
if media is None or not self.output_started:
|
| 375 |
+
return
|
| 376 |
+
|
| 377 |
+
try:
|
| 378 |
+
self._flush_output(reachy_mini)
|
| 379 |
+
media.stop_playing()
|
| 380 |
+
except Exception:
|
| 381 |
+
pass
|
| 382 |
+
finally:
|
| 383 |
+
self.output_started = False
|
| 384 |
+
|
| 385 |
+
def _push_chunk(self, reachy_mini: ReachyMini, samples: np.ndarray, start_time: float) -> bool:
|
| 386 |
+
if samples.size == 0:
|
| 387 |
+
return False
|
| 388 |
+
|
| 389 |
+
start_idx = max(0, int(start_time * self.SAMPLE_RATE))
|
| 390 |
+
chunk_len = max(1, int(self.CHUNK_SECONDS * self.SAMPLE_RATE))
|
| 391 |
+
end_idx = min(start_idx + chunk_len, len(samples))
|
| 392 |
+
if start_idx >= len(samples) or end_idx <= start_idx:
|
| 393 |
+
return False
|
| 394 |
+
|
| 395 |
+
chunk = samples[start_idx:end_idx]
|
| 396 |
+
try:
|
| 397 |
+
media = self._media(reachy_mini)
|
| 398 |
+
if media is None:
|
| 399 |
+
return False
|
| 400 |
+
media.push_audio_sample(chunk.astype(np.float32, copy=False))
|
| 401 |
+
self.next_push_time = end_idx / self.SAMPLE_RATE
|
| 402 |
+
return True
|
| 403 |
+
except Exception as exc:
|
| 404 |
+
print(f"[Audio robot] push_audio_sample a échoué : {exc}")
|
| 405 |
+
self.audio_available = False
|
| 406 |
+
return False
|
| 407 |
+
|
| 408 |
+
def update(self, reachy_mini: ReachyMini, state: dict, stop_event: threading.Event):
|
| 409 |
+
if stop_event.is_set() or not self.audio_available:
|
| 410 |
+
return
|
| 411 |
+
|
| 412 |
+
project_name = state.get("project_name")
|
| 413 |
+
playing = bool(state.get("playing", False))
|
| 414 |
+
updated_at = float(state.get("updated_at", time.time()))
|
| 415 |
+
current_time = max(0.0, float(state.get("current_time", 0.0)))
|
| 416 |
+
|
| 417 |
+
dirty = bool(state.get("dirty", False))
|
| 418 |
+
signature = (project_name, playing)
|
| 419 |
+
|
| 420 |
+
if not project_name or not playing:
|
| 421 |
+
if self.last_playing:
|
| 422 |
+
self._flush_output(reachy_mini)
|
| 423 |
+
self.last_playing = False
|
| 424 |
+
self.last_state_signature = signature
|
| 425 |
+
return
|
| 426 |
+
|
| 427 |
+
samples = self._load_audio(project_name)
|
| 428 |
+
if samples.size == 0:
|
| 429 |
+
self.last_playing = playing
|
| 430 |
+
self.last_state_signature = signature
|
| 431 |
+
return
|
| 432 |
+
|
| 433 |
+
self._start_output(reachy_mini)
|
| 434 |
+
if not self.output_started:
|
| 435 |
+
return
|
| 436 |
+
|
| 437 |
+
target_time = current_time + (time.time() - updated_at)
|
| 438 |
+
target_time = max(0.0, min(target_time, len(samples) / self.SAMPLE_RATE))
|
| 439 |
+
|
| 440 |
+
# Nouveau projet, play/reprise, ou seek : on repositionne l'audio.
|
| 441 |
+
# Les messages "sync" ne déclenchent pas de flush pour éviter de couper le son.
|
| 442 |
+
must_reposition = (
|
| 443 |
+
project_name != self.current_project
|
| 444 |
+
or (playing and not self.last_playing)
|
| 445 |
+
or (dirty and self.last_dirty_updated_at != updated_at)
|
| 446 |
+
)
|
| 447 |
+
if must_reposition:
|
| 448 |
+
self.current_project = project_name
|
| 449 |
+
self.next_push_time = target_time
|
| 450 |
+
self.last_dirty_updated_at = updated_at
|
| 451 |
+
self._flush_output(reachy_mini)
|
| 452 |
+
|
| 453 |
+
# Si le son prend trop d'avance/de retard, on se recale sur le temps vidéo.
|
| 454 |
+
if abs(self.next_push_time - target_time) > self.RESYNC_THRESHOLD_SECONDS:
|
| 455 |
+
self.next_push_time = target_time
|
| 456 |
+
self._flush_output(reachy_mini)
|
| 457 |
+
|
| 458 |
+
while (
|
| 459 |
+
not stop_event.is_set()
|
| 460 |
+
and self.next_push_time < target_time + self.QUEUE_AHEAD_SECONDS
|
| 461 |
+
):
|
| 462 |
+
if not self._push_chunk(reachy_mini, samples, self.next_push_time):
|
| 463 |
+
break
|
| 464 |
+
|
| 465 |
+
self.last_playing = playing
|
| 466 |
+
self.last_state_signature = signature
|
| 467 |
+
|
| 468 |
+
|
| 469 |
class Jericho(ReachyMiniApp):
|
| 470 |
custom_app_url: str | None = "http://0.0.0.0:8042"
|
| 471 |
+
request_media_backend: str | None = "default"
|
| 472 |
|
| 473 |
def __init__(self, **kwargs):
|
| 474 |
super().__init__(**kwargs)
|
|
|
|
| 645 |
if os.path.exists(project_path) and os.path.isdir(project_path):
|
| 646 |
try:
|
| 647 |
shutil.rmtree(project_path)
|
| 648 |
+
self.robot_audio.audio_cache.pop(project_name, None)
|
| 649 |
print(f"[{project_name}] Projet supprimé avec succès.")
|
| 650 |
return {"status": "success"}
|
| 651 |
except Exception as e:
|
|
|
|
| 823 |
state = dict(self.playback_state)
|
| 824 |
|
| 825 |
if state.get("stop_requested"):
|
| 826 |
+
self.robot_audio.update(reachy_mini, {**state, "playing": False}, stop_event)
|
| 827 |
reachy_mini.goto_target(create_head_pose(), antennas=[0.0, 0.0], duration=1.0)
|
| 828 |
with self.playback_lock:
|
| 829 |
self.playback_state["stop_requested"] = False
|
|
|
|
| 870 |
except TypeError:
|
| 871 |
reachy_mini.set_target(head=head_pose, antennas=antennas_target)
|
| 872 |
|
| 873 |
+
self.robot_audio.update(reachy_mini, state, stop_event)
|
| 874 |
+
|
| 875 |
if dirty:
|
| 876 |
with self.playback_lock:
|
| 877 |
|
|
|
|
| 880 |
|
| 881 |
time.sleep(0.01 if playing else 0.03)
|
| 882 |
|
| 883 |
+
self.robot_audio._stop_output(reachy_mini)
|
| 884 |
+
|
| 885 |
if __name__ == "__main__":
|
| 886 |
app = Jericho()
|
| 887 |
try:
|
jericho/static/main.js
CHANGED
|
@@ -6,6 +6,11 @@ const SYNC_INTERVAL_MS = 250;
|
|
| 6 |
const robotStatus = document.getElementById('robot-status');
|
| 7 |
const videoPlayer = document.getElementById('video-player');
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
function escapeHtml(value) {
|
| 10 |
return String(value)
|
| 11 |
.replaceAll('&', '&')
|
|
@@ -239,13 +244,16 @@ window.loadVideoPlayer = async function(name, url) {
|
|
| 239 |
document.getElementById('player-section').classList.remove('hidden');
|
| 240 |
document.getElementById('current-video-title').textContent = 'Projet : ' + name;
|
| 241 |
videoPlayer.src = url;
|
|
|
|
|
|
|
| 242 |
videoPlayer.load();
|
| 243 |
-
robotStatus.textContent = 'Vidéo chargée';
|
| 244 |
};
|
| 245 |
|
| 246 |
videoPlayer.addEventListener('play', () => {
|
| 247 |
if (!currentLoadedProject) return;
|
| 248 |
-
|
|
|
|
| 249 |
sendPlayback('play', { playing: true });
|
| 250 |
startSyncLoop();
|
| 251 |
});
|
|
|
|
| 6 |
const robotStatus = document.getElementById('robot-status');
|
| 7 |
const videoPlayer = document.getElementById('video-player');
|
| 8 |
|
| 9 |
+
// Le flux vidéo garde sa piste audio, mais le navigateur ne la joue pas.
|
| 10 |
+
// Le son est envoyé au haut-parleur du Reachy Mini depuis le serveur Python.
|
| 11 |
+
videoPlayer.muted = true;
|
| 12 |
+
videoPlayer.defaultMuted = true;
|
| 13 |
+
|
| 14 |
function escapeHtml(value) {
|
| 15 |
return String(value)
|
| 16 |
.replaceAll('&', '&')
|
|
|
|
| 244 |
document.getElementById('player-section').classList.remove('hidden');
|
| 245 |
document.getElementById('current-video-title').textContent = 'Projet : ' + name;
|
| 246 |
videoPlayer.src = url;
|
| 247 |
+
videoPlayer.muted = true;
|
| 248 |
+
videoPlayer.defaultMuted = true;
|
| 249 |
videoPlayer.load();
|
| 250 |
+
robotStatus.textContent = 'Vidéo chargée — son sur Reachy Mini';
|
| 251 |
};
|
| 252 |
|
| 253 |
videoPlayer.addEventListener('play', () => {
|
| 254 |
if (!currentLoadedProject) return;
|
| 255 |
+
videoPlayer.muted = true;
|
| 256 |
+
robotStatus.textContent = 'Lecture synchronisée — audio robot';
|
| 257 |
sendPlayback('play', { playing: true });
|
| 258 |
startSyncLoop();
|
| 259 |
});
|