File size: 7,448 Bytes
51bf86f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
#!/usr/bin/env python3
"""Checks the exported repo against the original PyTorch pipeline and against the FLEURS ground truth.

1. Tokenizer: capu.Tokenizer (the algorithm the Kotlin and Swift ports implement) against tokenizers' Unigram on
   every word of the test texts, as given and as corrected.
2. Pipeline: capu.Capu with the fp32 and the int8 model against out/reference.json (reference.py): share of texts
   with exactly the original's output.
3. Quality: punctuation (. , : ?) and capitalization F1 against the original FLEURS transcriptions, for the
   original, fp32 and int8.

Writes out/validation.json (read by publish.py), and fixtures for the Kotlin and Swift checks: out/fixtures/cases.tsv
(text, original, fp32 and int8 output) and out/fixtures/predictions.tsv (every int8 model call: token ids, word
offsets, label ids). Fails when int8 drifts too far from the original.

Usage:
  .venv/bin/python -I validate.py [--min-int8-f1-ratio 0.97]
"""

import argparse
import difflib
import json
import re
import sys
import time
import unicodedata
from pathlib import Path

from tokenizers import Tokenizer as HfTokenizer

sys.path.insert(0, str(Path(__file__).resolve().parent))  # -I drops the script's own directory
from capu import Capu, Tokenizer  # noqa: E402
from common import HERE, PUNCTUATION  # noqa: E402

REPO = HERE / "out" / "repo"


def check_tokenizer(rows: list[dict]) -> dict:
    ours = Tokenizer(REPO / "unigram.tsv")
    theirs = HfTokenizer.from_file(str(REPO / "tokenizer.json"))
    words = sorted({w for r in rows for key in ("text", "truth", "original") for w in r[key].split()})
    words += ["$START", "😀", "abc😀😀def", "fi", "Việt", "x²", "́a"]  # unk, NFKC, combining mark
    mismatches = []
    for word in words:
        expected = theirs.encode([word], is_pretokenized=True, add_special_tokens=False).ids
        if word == "$START":
            expected = [ours.start_id]  # an added token, matched before the unigram model
            got = [ours.start_id]
        else:
            got = ours.encode_word(word)
        if got != expected:
            mismatches.append((word, got, expected))
    for word, got, expected in mismatches[:10]:
        print(f"  tokenizer mismatch {word!r}: {got} != {expected}")
    print(f"tokenizer: {len(words) - len(mismatches)}/{len(words)} words identical")
    return {"words": len(words), "mismatches": len(mismatches)}


def word_labels(text: str) -> list[tuple[str, bool, str]]:
    """(lower-cased word, capitalized, following . , : ? or "") for each word of a text."""
    text = unicodedata.normalize("NFC", text)
    labels = []
    for word, after in re.findall(r"(\w+)([^\w\s]*)", text):
        mark = next((c for c in after if c in PUNCTUATION), "")
        labels.append((word.lower(), word[0].isupper(), mark))
    return labels


def scores(outputs: list[str], truths: list[str]) -> dict:
    """Micro F1 of punctuation marks and of capitalized words, over the words output and truth have in common."""
    counts = {"punct": [0, 0, 0], "upper": [0, 0, 0]}  # true positives, predicted, actual
    for output, truth in zip(outputs, truths):
        out, ref = word_labels(output), word_labels(truth)
        matcher = difflib.SequenceMatcher(None, [w for w, _, _ in out], [w for w, _, _ in ref], autojunk=False)
        for i, j, size in matcher.get_matching_blocks():
            for (_, o_up, o_mark), (_, r_up, r_mark) in zip(out[i:i + size], ref[j:j + size]):
                for key, o, r in (("punct", o_mark, r_mark), ("upper", o_up, r_up)):
                    counts[key][0] += bool(o) and o == r
                    counts[key][1] += bool(o)
                    counts[key][2] += bool(r)
    result = {}
    for key, (tp, predicted, actual) in counts.items():
        p, r = tp / max(predicted, 1), tp / max(actual, 1)
        result[f"{key}_f1"] = round(2 * p * r / max(p + r, 1e-9), 4)
    return result


class Recorder:
    """Wraps an InferenceSession and keeps every call's inputs and argmax labels."""

    def __init__(self, session):
        self.session, self.calls = session, {}

    def run(self, names, feed):
        result = self.session.run(names, feed)
        key = (",".join(map(str, feed["input_ids"][0])), ",".join(map(str, feed["input_offsets"][0])))
        self.calls[key] = ",".join(map(str, result[0][0].argmax(-1)))
        return result


def write_fixtures(rows: list[dict], outputs: dict[str, list[str]], calls: dict) -> None:
    fixtures = HERE / "out" / "fixtures"
    fixtures.mkdir(exist_ok=True)
    lines = ["\t".join((r["text"], r["original"], fp32, int8)) for r, fp32, int8 in zip(rows, outputs["fp32"], outputs["int8"])]
    (fixtures / "cases.tsv").write_text("\n".join(lines) + "\n", encoding="utf-8")
    lines = ["\t".join((ids, offsets, labels)) for (ids, offsets), labels in calls.items()]
    (fixtures / "predictions.tsv").write_text("\n".join(lines) + "\n", encoding="utf-8")
    print(f"wrote {fixtures} ({len(rows)} cases, {len(calls)} model calls)")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--min-int8-f1-ratio", type=float, default=0.97)
    args = parser.parse_args()
    rows = json.loads((HERE / "out" / "reference.json").read_text(encoding="utf-8"))
    graded = [r for r in rows if r["truth"]]
    long = sum(len(r["text"].split()) > 48 for r in rows)
    print(f"{len(rows)} texts, {long} longer than one 48-word chunk")

    report = {"texts": len(rows), "long_texts": long, "tokenizer": check_tokenizer(rows)}
    report["original"] = scores([r["original"] for r in graded], [r["truth"] for r in graded])
    print(f"original: {report['original']}")
    all_outputs = {}
    for quantized in (False, True):
        label = "int8" if quantized else "fp32"
        capu = Capu(REPO, quantized=quantized, threads=8)
        start = time.time()
        outputs = [capu(r["text"]) for r in rows]
        seconds = time.time() - start
        all_outputs[label] = outputs
        same = [o == r["original"] for o, r in zip(outputs, rows)]
        diffs = [(o, r["original"]) for o, r in zip(outputs, rows) if o != r["original"]]
        report[label] = {
            "identical": sum(same),
            "identical_long": sum(s for s, r in zip(same, rows) if len(r["text"].split()) > 48),
            **scores([o for o, r in zip(outputs, rows) if r["truth"]], [r["truth"] for r in graded]),
            "seconds": round(seconds, 1),
        }
        print(f"{label}: {report[label]}")
        for got, want in diffs[:3]:
            print(f"  got:  {got}\n  want: {want}")
    (HERE / "out" / "validation.json").write_text(json.dumps(report, indent=1) + "\n")
    capu.session = Recorder(capu.session)
    for r in rows:
        capu(r["text"])
    write_fixtures(rows, all_outputs, capu.session.calls)

    failed = []
    if report["tokenizer"]["mismatches"]:
        failed.append("tokenizer differs from tokenizers' Unigram")
    if report["fp32"]["identical"] < len(rows) * 0.99:
        failed.append("fp32 output differs from the original on more than 1% of texts")
    for key in ("punct_f1", "upper_f1"):
        if report["int8"][key] < report["original"][key] * args.min_int8_f1_ratio:
            failed.append(f"int8 {key} {report['int8'][key]} too far below the original's {report['original'][key]}")
    if failed:
        sys.exit("error: " + "; ".join(failed))


if __name__ == "__main__":
    main()