3v324v23 commited on
Commit
177e59b
·
2 Parent(s): 4c767d44667e2f

Merge branch 'main' of https://huggingface.co/spaces/Jarilo-Project/jericho

Browse files
Files changed (2) hide show
  1. jericho/main.py +217 -1
  2. jericho/static/main.js +10 -2
jericho/main.py CHANGED
@@ -256,9 +256,219 @@ class PlaybackPayload(BaseModel):
256
  playing: bool | None = None
257
  action: str = "sync"
258
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
259
  class Jericho(ReachyMiniApp):
260
  custom_app_url: str | None = "http://0.0.0.0:8042"
261
- request_media_backend: str | None = None
262
 
263
  def __init__(self, **kwargs):
264
  super().__init__(**kwargs)
@@ -435,6 +645,7 @@ class Jericho(ReachyMiniApp):
435
  if os.path.exists(project_path) and os.path.isdir(project_path):
436
  try:
437
  shutil.rmtree(project_path)
 
438
  print(f"[{project_name}] Projet supprimé avec succès.")
439
  return {"status": "success"}
440
  except Exception as e:
@@ -612,6 +823,7 @@ class Jericho(ReachyMiniApp):
612
  state = dict(self.playback_state)
613
 
614
  if state.get("stop_requested"):
 
615
  reachy_mini.goto_target(create_head_pose(), antennas=[0.0, 0.0], duration=1.0)
616
  with self.playback_lock:
617
  self.playback_state["stop_requested"] = False
@@ -658,6 +870,8 @@ class Jericho(ReachyMiniApp):
658
  except TypeError:
659
  reachy_mini.set_target(head=head_pose, antennas=antennas_target)
660
 
 
 
661
  if dirty:
662
  with self.playback_lock:
663
 
@@ -666,6 +880,8 @@ class Jericho(ReachyMiniApp):
666
 
667
  time.sleep(0.01 if playing else 0.03)
668
 
 
 
669
  if __name__ == "__main__":
670
  app = Jericho()
671
  try:
 
256
  playing: bool | None = None
257
  action: str = "sync"
258
 
259
+ class RobotAudioStreamer:
260
+ """Stream l'audio de la vidéo vers le haut-parleur Reachy Mini.
261
+
262
+ Le navigateur garde la vidéo pour l'image uniquement. L'audio est extrait
263
+ depuis output_annotated.mp4, converti en float32 16 kHz stéréo, puis poussé
264
+ par petits blocs dans mini.media.push_audio_sample().
265
+ """
266
+
267
+ SAMPLE_RATE = 16000
268
+ CHANNELS = 2
269
+ CHUNK_SECONDS = 0.08
270
+ QUEUE_AHEAD_SECONDS = 0.20
271
+ RESYNC_THRESHOLD_SECONDS = 0.25
272
+
273
+ def __init__(self, app: "Jericho"):
274
+ self.app = app
275
+ self.audio_cache: dict[str, np.ndarray] = {}
276
+ self.current_project: str | None = None
277
+ self.next_push_time = 0.0
278
+ self.output_started = False
279
+ self.last_state_signature = None
280
+ self.last_dirty_updated_at = None
281
+ self.last_playing = False
282
+ self.audio_available = True
283
+
284
+ def _project_video_path(self, project_name: str) -> str:
285
+ return os.path.join(self.app.projects_dir, project_name, "output_annotated.mp4")
286
+
287
+ def _load_audio(self, project_name: str) -> np.ndarray:
288
+ if project_name in self.audio_cache:
289
+ return self.audio_cache[project_name]
290
+
291
+ video_path = self._project_video_path(project_name)
292
+ if not os.path.exists(video_path):
293
+ self.audio_cache[project_name] = np.zeros((0, self.CHANNELS), dtype=np.float32)
294
+ return self.audio_cache[project_name]
295
+
296
+ cmd = [
297
+ "ffmpeg", "-v", "error",
298
+ "-i", video_path,
299
+ "-vn",
300
+ "-ac", str(self.CHANNELS),
301
+ "-ar", str(self.SAMPLE_RATE),
302
+ "-f", "f32le",
303
+ "pipe:1",
304
+ ]
305
+
306
+ try:
307
+ process = subprocess.run(cmd, capture_output=True, check=True)
308
+ audio = np.frombuffer(process.stdout, dtype=np.float32)
309
+ if audio.size == 0:
310
+ samples = np.zeros((0, self.CHANNELS), dtype=np.float32)
311
+ else:
312
+ usable = audio.size - (audio.size % self.CHANNELS)
313
+ samples = audio[:usable].reshape(-1, self.CHANNELS).astype(np.float32, copy=False)
314
+ except Exception as exc:
315
+ print(f"[Audio robot] Audio indisponible pour {project_name}: {exc}")
316
+ samples = np.zeros((0, self.CHANNELS), dtype=np.float32)
317
+
318
+ self.audio_cache[project_name] = samples
319
+ return samples
320
+
321
+ def _media(self, reachy_mini: ReachyMini):
322
+ return getattr(reachy_mini, "media", None)
323
+
324
+ def _start_output(self, reachy_mini: ReachyMini):
325
+ if self.output_started or not self.audio_available:
326
+ return
327
+
328
+ media = self._media(reachy_mini)
329
+ if media is None:
330
+ self.audio_available = False
331
+ print("[Audio robot] Backend média non disponible : audio robot désactivé.")
332
+ return
333
+
334
+ try:
335
+ media.start_playing()
336
+ # Réduit l'avance audio en file d'attente quand le backend le supporte.
337
+ for obj in (media, getattr(media, "audio", None)):
338
+ if obj is not None and hasattr(obj, "set_max_output_buffers"):
339
+ obj.set_max_output_buffers(3)
340
+ self.output_started = True
341
+ except Exception as exc:
342
+ self.audio_available = False
343
+ print(f"[Audio robot] Impossible de démarrer la sortie audio : {exc}")
344
+
345
+ def _flush_output(self, reachy_mini: ReachyMini):
346
+ """Vide le son déjà mis en queue pour éviter qu'il continue après pause/seek."""
347
+ media = self._media(reachy_mini)
348
+ if media is None or not self.output_started:
349
+ return
350
+
351
+ flushed = False
352
+ for obj in (media, getattr(media, "audio", None)):
353
+ if obj is None:
354
+ continue
355
+ for method_name in ("clear_player", "clear_output_buffer"):
356
+ method = getattr(obj, method_name, None)
357
+ if callable(method):
358
+ try:
359
+ method()
360
+ flushed = True
361
+ except Exception:
362
+ pass
363
+
364
+ if not flushed:
365
+ # Fallback générique : redémarrer la pipeline audio vide la plupart des backends.
366
+ try:
367
+ media.stop_playing()
368
+ media.start_playing()
369
+ except Exception:
370
+ pass
371
+
372
+ def _stop_output(self, reachy_mini: ReachyMini):
373
+ media = self._media(reachy_mini)
374
+ if media is None or not self.output_started:
375
+ return
376
+
377
+ try:
378
+ self._flush_output(reachy_mini)
379
+ media.stop_playing()
380
+ except Exception:
381
+ pass
382
+ finally:
383
+ self.output_started = False
384
+
385
+ def _push_chunk(self, reachy_mini: ReachyMini, samples: np.ndarray, start_time: float) -> bool:
386
+ if samples.size == 0:
387
+ return False
388
+
389
+ start_idx = max(0, int(start_time * self.SAMPLE_RATE))
390
+ chunk_len = max(1, int(self.CHUNK_SECONDS * self.SAMPLE_RATE))
391
+ end_idx = min(start_idx + chunk_len, len(samples))
392
+ if start_idx >= len(samples) or end_idx <= start_idx:
393
+ return False
394
+
395
+ chunk = samples[start_idx:end_idx]
396
+ try:
397
+ media = self._media(reachy_mini)
398
+ if media is None:
399
+ return False
400
+ media.push_audio_sample(chunk.astype(np.float32, copy=False))
401
+ self.next_push_time = end_idx / self.SAMPLE_RATE
402
+ return True
403
+ except Exception as exc:
404
+ print(f"[Audio robot] push_audio_sample a échoué : {exc}")
405
+ self.audio_available = False
406
+ return False
407
+
408
+ def update(self, reachy_mini: ReachyMini, state: dict, stop_event: threading.Event):
409
+ if stop_event.is_set() or not self.audio_available:
410
+ return
411
+
412
+ project_name = state.get("project_name")
413
+ playing = bool(state.get("playing", False))
414
+ updated_at = float(state.get("updated_at", time.time()))
415
+ current_time = max(0.0, float(state.get("current_time", 0.0)))
416
+
417
+ dirty = bool(state.get("dirty", False))
418
+ signature = (project_name, playing)
419
+
420
+ if not project_name or not playing:
421
+ if self.last_playing:
422
+ self._flush_output(reachy_mini)
423
+ self.last_playing = False
424
+ self.last_state_signature = signature
425
+ return
426
+
427
+ samples = self._load_audio(project_name)
428
+ if samples.size == 0:
429
+ self.last_playing = playing
430
+ self.last_state_signature = signature
431
+ return
432
+
433
+ self._start_output(reachy_mini)
434
+ if not self.output_started:
435
+ return
436
+
437
+ target_time = current_time + (time.time() - updated_at)
438
+ target_time = max(0.0, min(target_time, len(samples) / self.SAMPLE_RATE))
439
+
440
+ # Nouveau projet, play/reprise, ou seek : on repositionne l'audio.
441
+ # Les messages "sync" ne déclenchent pas de flush pour éviter de couper le son.
442
+ must_reposition = (
443
+ project_name != self.current_project
444
+ or (playing and not self.last_playing)
445
+ or (dirty and self.last_dirty_updated_at != updated_at)
446
+ )
447
+ if must_reposition:
448
+ self.current_project = project_name
449
+ self.next_push_time = target_time
450
+ self.last_dirty_updated_at = updated_at
451
+ self._flush_output(reachy_mini)
452
+
453
+ # Si le son prend trop d'avance/de retard, on se recale sur le temps vidéo.
454
+ if abs(self.next_push_time - target_time) > self.RESYNC_THRESHOLD_SECONDS:
455
+ self.next_push_time = target_time
456
+ self._flush_output(reachy_mini)
457
+
458
+ while (
459
+ not stop_event.is_set()
460
+ and self.next_push_time < target_time + self.QUEUE_AHEAD_SECONDS
461
+ ):
462
+ if not self._push_chunk(reachy_mini, samples, self.next_push_time):
463
+ break
464
+
465
+ self.last_playing = playing
466
+ self.last_state_signature = signature
467
+
468
+
469
  class Jericho(ReachyMiniApp):
470
  custom_app_url: str | None = "http://0.0.0.0:8042"
471
+ request_media_backend: str | None = "default"
472
 
473
  def __init__(self, **kwargs):
474
  super().__init__(**kwargs)
 
645
  if os.path.exists(project_path) and os.path.isdir(project_path):
646
  try:
647
  shutil.rmtree(project_path)
648
+ self.robot_audio.audio_cache.pop(project_name, None)
649
  print(f"[{project_name}] Projet supprimé avec succès.")
650
  return {"status": "success"}
651
  except Exception as e:
 
823
  state = dict(self.playback_state)
824
 
825
  if state.get("stop_requested"):
826
+ self.robot_audio.update(reachy_mini, {**state, "playing": False}, stop_event)
827
  reachy_mini.goto_target(create_head_pose(), antennas=[0.0, 0.0], duration=1.0)
828
  with self.playback_lock:
829
  self.playback_state["stop_requested"] = False
 
870
  except TypeError:
871
  reachy_mini.set_target(head=head_pose, antennas=antennas_target)
872
 
873
+ self.robot_audio.update(reachy_mini, state, stop_event)
874
+
875
  if dirty:
876
  with self.playback_lock:
877
 
 
880
 
881
  time.sleep(0.01 if playing else 0.03)
882
 
883
+ self.robot_audio._stop_output(reachy_mini)
884
+
885
  if __name__ == "__main__":
886
  app = Jericho()
887
  try:
jericho/static/main.js CHANGED
@@ -6,6 +6,11 @@ const SYNC_INTERVAL_MS = 250;
6
  const robotStatus = document.getElementById('robot-status');
7
  const videoPlayer = document.getElementById('video-player');
8
 
 
 
 
 
 
9
  function escapeHtml(value) {
10
  return String(value)
11
  .replaceAll('&', '&amp;')
@@ -239,13 +244,16 @@ window.loadVideoPlayer = async function(name, url) {
239
  document.getElementById('player-section').classList.remove('hidden');
240
  document.getElementById('current-video-title').textContent = 'Projet : ' + name;
241
  videoPlayer.src = url;
 
 
242
  videoPlayer.load();
243
- robotStatus.textContent = 'Vidéo chargée';
244
  };
245
 
246
  videoPlayer.addEventListener('play', () => {
247
  if (!currentLoadedProject) return;
248
- robotStatus.textContent = 'Lecture synchronisée';
 
249
  sendPlayback('play', { playing: true });
250
  startSyncLoop();
251
  });
 
6
  const robotStatus = document.getElementById('robot-status');
7
  const videoPlayer = document.getElementById('video-player');
8
 
9
+ // Le flux vidéo garde sa piste audio, mais le navigateur ne la joue pas.
10
+ // Le son est envoyé au haut-parleur du Reachy Mini depuis le serveur Python.
11
+ videoPlayer.muted = true;
12
+ videoPlayer.defaultMuted = true;
13
+
14
  function escapeHtml(value) {
15
  return String(value)
16
  .replaceAll('&', '&amp;')
 
244
  document.getElementById('player-section').classList.remove('hidden');
245
  document.getElementById('current-video-title').textContent = 'Projet : ' + name;
246
  videoPlayer.src = url;
247
+ videoPlayer.muted = true;
248
+ videoPlayer.defaultMuted = true;
249
  videoPlayer.load();
250
+ robotStatus.textContent = 'Vidéo chargée — son sur Reachy Mini';
251
  };
252
 
253
  videoPlayer.addEventListener('play', () => {
254
  if (!currentLoadedProject) return;
255
+ videoPlayer.muted = true;
256
+ robotStatus.textContent = 'Lecture synchronisée — audio robot';
257
  sendPlayback('play', { playing: true });
258
  startSyncLoop();
259
  });