| """Fill REPORT.md tables from results/*.json. python make_report.py <eval_summary.json> <asr_scores.json> <llm_summary.json> -> prints markdown blocks""" |
| import json, sys |
| ev, asr, llm = (json.load(open(f)) for f in sys.argv[1:4]) |
| L = ["eng", "pcm", "ibo", "yor", "hau", "cs"] |
| o = ["### ASR (229 held-out clips; WER / CER; CER without diacritics in brackets)", "", |
| "| model | " + " | ".join(L) + " | latency p50 / p90 per clip | CS mix TVD after ASR (rules / +GlotLID windows) |", "|---" * (len(L) + 3) + "|"] |
| for k, v in asr.items(): |
| if k.startswith("_"): continue |
| o.append(f"| {k} | " + " | ".join(f"{v[l]['wer']:.2f} / {v[l]['cer']:.2f} ({v[l]['cer_nodiac']:.2f})" for l in L) + |
| f" | {v['latency']['p50_s']*1000:.0f} / {v['latency']['p90_s']*1000:.0f} ms | {v['cs_mix_tvd_after_asr'].get('none')} / {v['cs_mix_tvd_after_asr'].get('glotlid')} |") |
| o += ["", f"LID on reference CS text (upper bound): word accuracy {asr['_lid_on_reference_text']['glotlid']['word_acc']}, mix TVD {asr['_lid_on_reference_text']['glotlid']['mix_tvd']}.", ""] |
| o += ["### Reply LLM (28 prompts: 16 mixed/greeting/Pidgin/English written prompts + 12 native WAXAL sentences ibo/yor/hau)", "", |
| "| model | subset | first candidate passes | regen rate | fallback | mean mix TVD | GlotLID agrees with span tag | judge 1-5 (back-translated) | LLM s (n=3) |", "|---|---|---|---|---|---|---|---|---|"] |
| for r in llm: |
| for sub in ("mixed", "native", "all"): |
| g = r[sub] |
| o.append(f"| {r['model']} | {sub} | {g['first_cand_ok']} | {g['regen']} | {g['fallback']} | {g['tvd']} | {g['glot_ok']} | {g['judge']} | {g['llm_s']} |") |
| for name in ("batch_eval", "demos"): |
| e = ev[name]; la = e["latency"] |
| o += ["", f"### Latency, {name.replace('_', ' ')} ({la['latency_s']['n']} turns; seconds, p50 / p90)", "", "| stage | p50 | p90 |", "|---|---|---|"] |
| for k, lab in [("eot_wait_s", "end-of-turn wait (VAD silence)"), ("asr_s", "ASR final"), ("mix_s", "LID + mix"), ("llm_s", "LLM + validator rounds"), ("tts_first_s", "TTS first chunk"), |
| ("compute_to_first_audio_s", "compute total (ASR->first chunk)"), ("latency_s", "**end of user speech -> first agent audio**")]: |
| o.append(f"| {lab} | {la[k]['p50']} | {la[k]['p90']} |") |
| m = e["mix"] |
| o += ["", f"Mix adherence ({name}): mean TVD(target, reply) {m['tvd_mean']} (p90 {m['tvd_p90']}); first candidate passes {m['first_candidate_ok']}; regenerated {m['regen_rate']}; fallback {m['fallback_rate']}" + (f" (safe phrasebook {m.get('safe_fallback_rate')})" if 'safe_fallback_rate' in m else "") + f"; {m['multi_lang_targets']}/{m['n']} turns had a multi-language target."] |
| b = ev["batch_eval"]["barge_in"] |
| o += ["", f"Barge-in (batch eval, n={b['n']}): user speech onset -> agent audio stopped p50 {b['reaction_s']['p50']} s, p90 {b['reaction_s']['p90']} s (VAD detection p50 {b['detected_minus_onset']['p50']} s + 40 ms output flush)."] |
| print("\n".join(o)) |
|
|
