Spaces:
Paused
Paused
JasonZhai-9936 Claude Opus 5.5 commited on
Commit ·
059172b
1
Parent(s): 8458b91
Add /api/timeline: compact per-turn latency/sync summary (last 10 turns, ?full=1 for every line) so results can be copied without scraping the Space log viewer
Browse files- app.py +17 -1
- float_streamer.py +2 -0
- timeline.py +70 -3
app.py
CHANGED
|
@@ -4,7 +4,7 @@ Avatar profile system: each avatar has ref.png, persona.txt, idlevideos/
|
|
| 4 |
Chunked pipeline: LLM (once) → split sentences → TTS+FLOAT per chunk → stream to frontend
|
| 5 |
"""
|
| 6 |
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
| 7 |
-
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
|
| 8 |
from pathlib import Path
|
| 9 |
import mimetypes
|
| 10 |
import os
|
|
@@ -901,6 +901,14 @@ async def interrupt():
|
|
| 901 |
return JSONResponse({"status": "ok"})
|
| 902 |
|
| 903 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 904 |
@app.post("/api/timing")
|
| 905 |
async def client_timing(request: Request):
|
| 906 |
"""Client-side playback milestones, logged into the same turn timeline so
|
|
@@ -916,6 +924,7 @@ async def client_timing(request: Request):
|
|
| 916 |
f"(waited {b.get('queued_ms', 0):.0f}ms for previous clip + {b.get('load_ms', 0):.0f}ms loading"
|
| 917 |
f"{', preloaded' if b.get('preloaded') else ', NOT preloaded'}) "
|
| 918 |
f"= how far audio lags its video", turn)
|
|
|
|
| 919 |
elif ev == "audio_end":
|
| 920 |
timeline.mark(f"{tag} CLIENT audio ended (played {b.get('played_ms', 0)/1000:.2f}s)", turn)
|
| 921 |
return JSONResponse({"status": "ok"})
|
|
@@ -964,6 +973,8 @@ async def ws_video(websocket: WebSocket):
|
|
| 964 |
await websocket.send_json(item)
|
| 965 |
if item.get("type") == "speech_start":
|
| 966 |
timeline.mark(f"{item.get('tag')} speech_start sent to client — its video frames follow now", item.get("turn"))
|
|
|
|
|
|
|
| 967 |
elif item.get("type") == "speech_end":
|
| 968 |
timeline.mark("speech_end sent to client", item.get("turn"))
|
| 969 |
continue
|
|
@@ -1084,6 +1095,9 @@ async def _chat_stream_process_chunk(idx: int, total: int, sentence: str, stream
|
|
| 1084 |
timeline.mark(
|
| 1085 |
f"{tag} TTS done in {t_b-t_a:.2f}s ({audio_duration:.1f}s audio)" if audio_path
|
| 1086 |
else f"{tag} TTS FAILED after {t_b-t_a:.2f}s", turn)
|
|
|
|
|
|
|
|
|
|
| 1087 |
|
| 1088 |
if gen_id != _generation_id:
|
| 1089 |
timeline.mark(f"{tag} stale (interrupted mid-flight) — discarding", turn)
|
|
@@ -1158,6 +1172,7 @@ async def chat_stream(request: Request):
|
|
| 1158 |
clean_text = llm_result.get('clean_text', reply_text)
|
| 1159 |
t2 = _time.time()
|
| 1160 |
timeline.mark(f"LLM done in {t2-t1:.2f}s: \"{clean_text[:80]}...\"", turn)
|
|
|
|
| 1161 |
|
| 1162 |
# 2. Split into short chunks (~12 words each) so the first chunk's TTS
|
| 1163 |
# call is small and fast instead of synthesizing the entire reply.
|
|
@@ -1166,6 +1181,7 @@ async def chat_stream(request: Request):
|
|
| 1166 |
sentences = [reply_text]
|
| 1167 |
total = len(sentences)
|
| 1168 |
timeline.mark(f"reply split into {total} chunk(s)", turn)
|
|
|
|
| 1169 |
|
| 1170 |
from float_streamer import get_streamer
|
| 1171 |
streamer = get_streamer()
|
|
|
|
| 4 |
Chunked pipeline: LLM (once) → split sentences → TTS+FLOAT per chunk → stream to frontend
|
| 5 |
"""
|
| 6 |
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
| 7 |
+
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, PlainTextResponse
|
| 8 |
from pathlib import Path
|
| 9 |
import mimetypes
|
| 10 |
import os
|
|
|
|
| 901 |
return JSONResponse({"status": "ok"})
|
| 902 |
|
| 903 |
|
| 904 |
+
@app.get("/api/timeline")
|
| 905 |
+
async def get_timeline(full: int = 0):
|
| 906 |
+
"""Compact per-turn latency/sync summary for the last 10 turns — open in a
|
| 907 |
+
browser tab and copy, instead of scraping the Space's log viewer.
|
| 908 |
+
?full=1 adds every timeline line under each turn."""
|
| 909 |
+
return PlainTextResponse(timeline.summary(full=bool(full)))
|
| 910 |
+
|
| 911 |
+
|
| 912 |
@app.post("/api/timing")
|
| 913 |
async def client_timing(request: Request):
|
| 914 |
"""Client-side playback milestones, logged into the same turn timeline so
|
|
|
|
| 924 |
f"(waited {b.get('queued_ms', 0):.0f}ms for previous clip + {b.get('load_ms', 0):.0f}ms loading"
|
| 925 |
f"{', preloaded' if b.get('preloaded') else ', NOT preloaded'}) "
|
| 926 |
f"= how far audio lags its video", turn)
|
| 927 |
+
timeline.stat("lags", (b.get("since_start_ms", 0), bool(b.get("preloaded"))), turn)
|
| 928 |
elif ev == "audio_end":
|
| 929 |
timeline.mark(f"{tag} CLIENT audio ended (played {b.get('played_ms', 0)/1000:.2f}s)", turn)
|
| 930 |
return JSONResponse({"status": "ok"})
|
|
|
|
| 973 |
await websocket.send_json(item)
|
| 974 |
if item.get("type") == "speech_start":
|
| 975 |
timeline.mark(f"{item.get('tag')} speech_start sent to client — its video frames follow now", item.get("turn"))
|
| 976 |
+
if str(item.get("tag", "")).startswith("c1/"):
|
| 977 |
+
timeline.stat("video_start", timeline.elapsed(), item.get("turn"))
|
| 978 |
elif item.get("type") == "speech_end":
|
| 979 |
timeline.mark("speech_end sent to client", item.get("turn"))
|
| 980 |
continue
|
|
|
|
| 1095 |
timeline.mark(
|
| 1096 |
f"{tag} TTS done in {t_b-t_a:.2f}s ({audio_duration:.1f}s audio)" if audio_path
|
| 1097 |
else f"{tag} TTS FAILED after {t_b-t_a:.2f}s", turn)
|
| 1098 |
+
if idx == 0:
|
| 1099 |
+
timeline.stat("c1_tts", t_b - t_a, turn)
|
| 1100 |
+
timeline.stat("c1_audio", audio_duration, turn)
|
| 1101 |
|
| 1102 |
if gen_id != _generation_id:
|
| 1103 |
timeline.mark(f"{tag} stale (interrupted mid-flight) — discarding", turn)
|
|
|
|
| 1172 |
clean_text = llm_result.get('clean_text', reply_text)
|
| 1173 |
t2 = _time.time()
|
| 1174 |
timeline.mark(f"LLM done in {t2-t1:.2f}s: \"{clean_text[:80]}...\"", turn)
|
| 1175 |
+
timeline.stat("llm", t2 - t1, turn)
|
| 1176 |
|
| 1177 |
# 2. Split into short chunks (~12 words each) so the first chunk's TTS
|
| 1178 |
# call is small and fast instead of synthesizing the entire reply.
|
|
|
|
| 1181 |
sentences = [reply_text]
|
| 1182 |
total = len(sentences)
|
| 1183 |
timeline.mark(f"reply split into {total} chunk(s)", turn)
|
| 1184 |
+
timeline.stat("chunks", total, turn)
|
| 1185 |
|
| 1186 |
from float_streamer import get_streamer
|
| 1187 |
streamer = get_streamer()
|
float_streamer.py
CHANGED
|
@@ -398,6 +398,8 @@ class FloatStreamer:
|
|
| 398 |
# rendering is AHEAD of playback, not behind it.
|
| 399 |
took = t_dec_done - t0 - t_wait
|
| 400 |
slow = f" ⚠ SLOWER THAN REAL TIME ({took/video_s:.2f}x)" if video_s and took > video_s else ""
|
|
|
|
|
|
|
| 401 |
timeline.mark(
|
| 402 |
f"{tag} rendered {frames_pushed} frames ({video_s:.2f}s video) in {took:.2f}s "
|
| 403 |
f"[ode {t_ode:.2f}s, decode {t_dec_done-t_dec-t_wait:.2f}s, waited {t_wait:.2f}s for buffer space]{slow}", turn)
|
|
|
|
| 398 |
# rendering is AHEAD of playback, not behind it.
|
| 399 |
took = t_dec_done - t0 - t_wait
|
| 400 |
slow = f" ⚠ SLOWER THAN REAL TIME ({took/video_s:.2f}x)" if video_s and took > video_s else ""
|
| 401 |
+
if slow:
|
| 402 |
+
timeline.stat("slow_renders", 1, turn)
|
| 403 |
timeline.mark(
|
| 404 |
f"{tag} rendered {frames_pushed} frames ({video_s:.2f}s video) in {took:.2f}s "
|
| 405 |
f"[ode {t_ode:.2f}s, decode {t_dec_done-t_dec-t_wait:.2f}s, waited {t_wait:.2f}s for buffer space]{slow}", turn)
|
timeline.py
CHANGED
|
@@ -8,7 +8,12 @@ where T3 is the turn number and +1.42s is time since the user's message
|
|
| 8 |
arrived, so a copied log reads top-to-bottom as an ordered breakdown of where
|
| 9 |
the time went. Shared by app.py (request side) and float_streamer.py
|
| 10 |
(render thread), hence its own module.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
"""
|
|
|
|
| 12 |
import logging
|
| 13 |
import time
|
| 14 |
|
|
@@ -16,6 +21,19 @@ logger = logging.getLogger("timeline")
|
|
| 16 |
|
| 17 |
_turn = 0
|
| 18 |
_t0 = time.time()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
def begin(label: str) -> int:
|
|
@@ -23,7 +41,10 @@ def begin(label: str) -> int:
|
|
| 23 |
global _turn, _t0
|
| 24 |
_turn += 1
|
| 25 |
_t0 = time.time()
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
| 27 |
return _turn
|
| 28 |
|
| 29 |
|
|
@@ -31,11 +52,57 @@ def current() -> int:
|
|
| 31 |
return _turn
|
| 32 |
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
def mark(msg: str, turn: int = None):
|
| 35 |
"""Log one stage of the current turn. A mark tagged with an older turn
|
| 36 |
(e.g. a late client report after barge-in) is labelled as such, since its
|
| 37 |
offset is measured against the newer turn's start."""
|
| 38 |
if turn is not None and turn != _turn:
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
return
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
arrived, so a copied log reads top-to-bottom as an ordered breakdown of where
|
| 9 |
the time went. Shared by app.py (request side) and float_streamer.py
|
| 10 |
(render thread), hence its own module.
|
| 11 |
+
|
| 12 |
+
The last few turns are also kept in memory (lines + key stats) so /api/timeline
|
| 13 |
+
can serve a compact per-turn summary — the Space's log viewer is too long to
|
| 14 |
+
copy/paste whole.
|
| 15 |
"""
|
| 16 |
+
import collections
|
| 17 |
import logging
|
| 18 |
import time
|
| 19 |
|
|
|
|
| 21 |
|
| 22 |
_turn = 0
|
| 23 |
_t0 = time.time()
|
| 24 |
+
_history = collections.deque(maxlen=10) # recent turns: {turn, label, lines, stats}
|
| 25 |
+
_by_turn = {}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _entry(turn):
|
| 29 |
+
e = _by_turn.get(turn)
|
| 30 |
+
if e is None:
|
| 31 |
+
e = {"turn": turn, "label": "", "lines": [], "stats": {"lags": []}}
|
| 32 |
+
if len(_history) == _history.maxlen:
|
| 33 |
+
_by_turn.pop(_history[0]["turn"], None)
|
| 34 |
+
_history.append(e)
|
| 35 |
+
_by_turn[turn] = e
|
| 36 |
+
return e
|
| 37 |
|
| 38 |
|
| 39 |
def begin(label: str) -> int:
|
|
|
|
| 41 |
global _turn, _t0
|
| 42 |
_turn += 1
|
| 43 |
_t0 = time.time()
|
| 44 |
+
_entry(_turn)["label"] = label
|
| 45 |
+
line = f"[T{_turn} + 0.00s] ===== TURN START: {label}"
|
| 46 |
+
_entry(_turn)["lines"].append(line)
|
| 47 |
+
logger.info(line)
|
| 48 |
return _turn
|
| 49 |
|
| 50 |
|
|
|
|
| 52 |
return _turn
|
| 53 |
|
| 54 |
|
| 55 |
+
def elapsed() -> float:
|
| 56 |
+
return time.time() - _t0
|
| 57 |
+
|
| 58 |
+
|
| 59 |
def mark(msg: str, turn: int = None):
|
| 60 |
"""Log one stage of the current turn. A mark tagged with an older turn
|
| 61 |
(e.g. a late client report after barge-in) is labelled as such, since its
|
| 62 |
offset is measured against the newer turn's start."""
|
| 63 |
if turn is not None and turn != _turn:
|
| 64 |
+
line = f"[T{turn} (late, now T{_turn})] {msg}"
|
| 65 |
+
target = turn
|
| 66 |
+
else:
|
| 67 |
+
line = f"[T{_turn} +{elapsed():6.2f}s] {msg}"
|
| 68 |
+
target = _turn
|
| 69 |
+
_entry(target)["lines"].append(line)
|
| 70 |
+
logger.info(line)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def stat(key: str, value, turn: int = None):
|
| 74 |
+
"""Record a summary number for a turn (ignored for stale turns)."""
|
| 75 |
+
if turn is not None and turn != _turn:
|
| 76 |
return
|
| 77 |
+
stats = _entry(_turn)["stats"]
|
| 78 |
+
if key == "lags":
|
| 79 |
+
stats["lags"].append(value)
|
| 80 |
+
elif key == "slow_renders":
|
| 81 |
+
stats["slow_renders"] = stats.get("slow_renders", 0) + 1
|
| 82 |
+
else:
|
| 83 |
+
stats.setdefault(key, value) # first occurrence wins
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def summary(full: bool = False) -> str:
|
| 87 |
+
out = []
|
| 88 |
+
for e in list(_history):
|
| 89 |
+
if e["turn"] == 0 and not e["lines"]:
|
| 90 |
+
continue
|
| 91 |
+
s = e["stats"]
|
| 92 |
+
parts = [f"T{e['turn']} {e['label'][:60] or '(startup / greeting)'}"]
|
| 93 |
+
if "llm" in s: parts.append(f"LLM {s['llm']:.2f}s")
|
| 94 |
+
if "chunks" in s: parts.append(f"{s['chunks']} chunks")
|
| 95 |
+
if "c1_tts" in s: parts.append(f"c1 TTS {s['c1_tts']:.2f}s ({s.get('c1_audio', 0):.1f}s audio)")
|
| 96 |
+
if "video_start" in s: parts.append(f"video starts +{s['video_start']:.2f}s")
|
| 97 |
+
if "video_start" in s and s["lags"]:
|
| 98 |
+
# c1's lag is measured client-side, so this avoids the report's own network delay
|
| 99 |
+
parts.append(f"audio starts +{s['video_start'] + s['lags'][0][0] / 1000:.2f}s")
|
| 100 |
+
if s["lags"]:
|
| 101 |
+
parts.append("A/V lag per chunk: " + " ".join(
|
| 102 |
+
f"{ms:.0f}ms{'' if pre else '(not preloaded)'}" for ms, pre in s["lags"]))
|
| 103 |
+
parts.append(f"slow renders: {s.get('slow_renders', 0)}")
|
| 104 |
+
out.append(" | ".join(parts))
|
| 105 |
+
if full:
|
| 106 |
+
out += [" " + l for l in e["lines"]]
|
| 107 |
+
out.append("")
|
| 108 |
+
return "\n".join(out) or "(no turns yet)"
|