| """Aggregate offline batch-eval + demo logs into results/eval_summary.json (latency per stage p50/p90, barge-in, mix adherence).""" |
| import json, glob, sys, statistics as st, os |
| base = sys.argv[1] if len(sys.argv) > 1 else "results/out1" |
| demo_dir = sys.argv[2] if len(sys.argv) > 2 else f"{base}/demos" |
| def q(x, p): |
| x = sorted(v for v in x if v is not None); return round(x[min(len(x) - 1, int(p * len(x)))], 3) if x else None |
| def pq(x): return dict(p50=q(x, .5), p90=q(x, .9), n=len([v for v in x if v is not None])) |
| E = json.load(open(f"{base}/eval/eval_turns.json")) |
| turns = [t for e in E for t in e["turns"]] |
| D = [t for f in sorted(glob.glob(f"{demo_dir}/*.json")) for t in json.load(open(f))["turns"]] |
| def lat(T): |
| return {k: pq([t.get(k) for t in T]) for k in ("latency_s", "eot_wait_s", "asr_s", "mix_s", "llm_s", "tts_first_s", "compute_to_first_audio_s")} |
| def mixstats(T): |
| n = len(T) |
| return dict(n=n, tvd_mean=round(st.mean(t["tvd"] for t in T if t["tvd"] is not None), 3), tvd_p90=q([t["tvd"] for t in T], .9), |
| first_candidate_ok=round(sum(t["first_cand_ok"] for t in T) / n, 3), regen_rate=round(sum(t["how"].startswith("regen") for t in T) / n, 3), |
| fallback_rate=round(sum(t["how"] in ("best_failed", "plain_matrix", "safe_phrasebook") for t in T) / n, 3), safe_fallback_rate=round(sum(t["how"] == "safe_phrasebook" for t in T) / n, 3), |
| multi_lang_targets=sum(len(t["target"]["mix"]) > 1 for t in T)) |
| by_lang = {} |
| for e in E: |
| t = e["turns"][0]; by_lang.setdefault(e["lang"], []).append(t) |
| barge = [t["barge"] for t in turns if t.get("barge")] |
| out = dict( |
| batch_eval=dict(desc="40 real held-out clips (WAXAL test ibo/yor/hau/pcm, FLEURS val eng, concatenated code-switch), each user turn followed by a barge-in clip 1.0 s into the reply (that clip is answered too)", |
| latency=lat(turns), mix=mixstats(turns), per_lang_first_turn={l: dict(lat=pq([t["latency_s"] for t in T]), **mixstats(T)) for l, T in by_lang.items()}, |
| barge_in=dict(n=len(barge), reaction_s=pq([b["reaction_s"] for b in barge]), detected_minus_onset=pq([b["detected"] - b["user_onset"] for b in barge])), |
| backchannels=sum(len(t.get("backchannels", [])) for t in turns)), |
| demos=dict(n_turns=len(D), latency=lat(D), mix=mixstats(D), barge_in=[t["barge"] for t in D if t.get("barge")], backchannels=sum(len(t.get("backchannels", [])) for t in D))) |
| os.makedirs("results", exist_ok=True); json.dump(out, open("results/eval_summary.json", "w"), indent=1); print(json.dumps(out, indent=1)) |
|
|
