"""Score both shipped prompts on ONE sentence set, so the card's table is real.

The two voices were confirmed in separate runs against different held-out sets
-- male on the v4 corpus holdout, female on the v3 one. Each of those runs was
internally sound: the new prompt and the one it replaced were paired on the same
sentences, which is what makes +0.728 and +0.527 trustworthy. But the card shows
the two voices side by side, and that comparison was never measured.

So this re-scores both against the same 120 held-out sentences, three seeds,
n=360 each. Nothing else changes: same checkpoint, same sampler constants, same
prompts that ship today. The output is a table whose columns can honestly be
read against one another.

Sentences come from the preserved v3 text holdout, withheld from the corpus the
model trained on, so they measure intelligibility rather than recall.
"""

from __future__ import annotations

import json
import pathlib
import random
import statistics
import sys

REPO = pathlib.Path("/workspace/oron-tts")
sys.path.insert(0, str(REPO))
sys.path.insert(0, str(REPO / "scripts"))

SEEDS = (0, 1, 2)
N_SENTENCES = 120
HUMAN_CER_BASELINE = 0.123


def cer_counts(ref, hyp):
    r, h = list(ref), list(hyp)
    if not r:
        return (len(h), 0)
    prev = list(range(len(h) + 1))
    for i, rc in enumerate(r, 1):
        cur = [i]
        for j, hc in enumerate(h, 1):
            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (rc != hc)))
        prev = cur
    return prev[-1], len(r)


def micro_cer(c):
    e = sum(a for a, _ in c)
    n = sum(b for _, b in c)
    return e / n if n else 0.0


def boot(vals, stat, resamples=2000, seed=0):
    items = list(vals)
    if len(items) < 2:
        v = stat(items) if items else float("nan")
        return v, v, v
    rng = random.Random(seed)
    n = len(items)
    d = sorted(stat([items[rng.randrange(n)] for _ in range(n)]) for _ in range(resamples))
    return stat(items), d[int(0.025 * resamples)], d[int(0.975 * resamples)]


def paired(a, b, stat, resamples=2000, seed=0):
    xs, ys = list(a), list(b)
    n = min(len(xs), len(ys))
    if n < 2:
        return None
    xs, ys = xs[:n], ys[:n]
    rng = random.Random(seed)
    d = []
    for _ in range(resamples):
        idx = [rng.randrange(n) for _ in range(n)]
        d.append(stat([xs[i] for i in idx]) - stat([ys[i] for i in idx]))
    d.sort()
    lo, hi = d[int(0.025 * resamples)], d[int(0.975 * resamples)]
    return {"diff": stat(xs) - stat(ys), "lo": lo, "hi": hi,
            "separated": lo > 0 or hi < 0, "n": n}


def main() -> None:
    import os

    from f5_tts.api import F5TTS
    from huggingface_hub import HfApi, hf_hub_download

    from eval_mn import synthesise
    from oron_tts.eval import MongolianASR, bandwidth_hz, utmos
    from oron_tts.eval.metrics import normalize_for_scoring

    lines = [ln.strip() for ln in pathlib.Path(
        hf_hub_download("btsee/oron-tts", "eval/cv_eval_sentences.txt")
    ).read_text(encoding="utf-8").splitlines() if ln.strip()]
    sents = lines[1::2][:N_SENTENCES]
    print(f"one common set: {len(sents)} sentences x {len(SEEDS)} seeds "
          f"= {len(sents) * len(SEEDS)} per voice")

    ckpt = hf_hub_download("btsee/oron-tts", "model.safetensors")
    vocab = hf_hub_download("btsee/oron-tts", "vocab.txt")
    asr = MongolianASR(device="cuda")

    out = {}
    for gender in ("male", "female"):
        wav = pathlib.Path(hf_hub_download("btsee/oron-tts", f"voices/{gender}.wav",
                                           force_download=True))
        rtxt = pathlib.Path(hf_hub_download("btsee/oron-tts", f"voices/{gender}.txt",
                                            force_download=True)
                            ).read_text(encoding="utf-8").strip()
        f5 = F5TTS(model="F5TTS_v1_Base", ckpt_file=ckpt, vocab_file=vocab,
                   device="cuda", use_ema=False)
        counts, moses, bws = [], [], []
        for k, text in enumerate(sents):
            for seed in SEEDS:
                try:
                    w, sr = synthesise(f5, text, wav, rtxt, seed)
                except Exception as exc:
                    print(f"    synth failed: {str(exc)[:70]}")
                    continue
                counts.append(cer_counts(normalize_for_scoring(text),
                                         normalize_for_scoring(asr.transcribe(w, sr))))
                bws.append(bandwidth_hz(w, sr))
                try:
                    moses.append(utmos(w, sr))
                except Exception:
                    pass
            if k % 30 == 0:
                print(f"    {gender} {k}/{len(sents)}", flush=True)
        cp, clo, chi = boot(counts, micro_cer)
        up, ulo, uhi = boot(moses, statistics.fmean)
        out[gender] = {"cer_micro": cp, "cer_lo": clo, "cer_hi": chi,
                       "utmos": up, "utmos_lo": ulo, "utmos_hi": uhi,
                       "utmos_values": moses, "n": len(counts),
                       "bandwidth_median": statistics.median(bws) if bws else None}
        print(f"  {gender:<7} UTMOS {up:.3f} [{ulo:.3f}, {uhi:.3f}]  "
              f"CER {cp:.4f} [{clo:.4f}, {chi:.4f}]  n={len(counts)}")
        del f5
        import torch
        torch.cuda.empty_cache()

    # Now the comparison the card's table implies, actually measured.
    d = paired(out["male"]["utmos_values"], out["female"]["utmos_values"],
               statistics.fmean)
    verdict = ("separated" if d["separated"]
               else "NOT separated -- the two voices score alike on this set")
    print(f"\n  male - female UTMOS {d['diff']:+.3f} [{d['lo']:+.3f}, {d['hi']:+.3f}]  "
          f"n={d['n']}  {verdict}")
    out["_male_vs_female"] = d

    payload = {"shipped-v4-common-set": {
        g: {k: v for k, v in out[g].items() if k != "utmos_values"}
        for g in ("male", "female")}, "male_vs_female": d,
        "sentences": len(sents), "seeds": list(SEEDS)}
    p = pathlib.Path("/workspace/common_eval.json")
    p.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
    HfApi(token=os.environ["HF_TOKEN"]).upload_file(
        path_or_fileobj=str(p), path_in_repo="eval/common_eval.json",
        repo_id="btsee/oron-tts",
        commit_message="Both prompts on one sentence set, so the columns compare")
    print("published eval/common_eval.json")


if __name__ == "__main__":
    main()