keep only the shipped model and remove earlier training runs

train_realword.py

@@ -0,0 +1,506 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = [
+#   "torch>=2.6,<2.8",
+#   "datasets>=3.0",
+#   "numpy<3",
+#   "onnx>=1.17",
+#   "onnxruntime>=1.20",
+#   "huggingface_hub>=0.30",
+# ]
+# ///
+"""Train a small model that spots correctly spelled wrong words, like "don" typed for "do"."""
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import time
+import urllib.request
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import numpy as np
+import onnxruntime as ort
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from datasets import load_dataset
+from huggingface_hub import HfApi
+from onnxruntime.quantization import QuantType, quantize_dynamic
+
+LEFT_WINDOW = 16
+RIGHT_WINDOW = 4
+MAX_NEIGHBORS = 8
+CANDIDATES = MAX_NEIGHBORS + 1
+MIN_TYPED_LENGTH = 3
+
+PAD, UNK, NUMBER, START, NAME = 0, 1, 2, 3, 4
+SPECIAL_TOKENS = ["<pad>", "<unk>", "<num>", "<s>", "<name>"]
+
+REAL_MISTAKE_SHARE = 0.6
+HOMOPHONE_WEIGHT = 20
+MIN_REAL_MISTAKE_COUNT = 2
+HOMOPHONES = [
+    ("their", "there", "they're"), ("your", "you're"), ("its", "it's"), ("too", "two"), ("then", "than"),
+    ("were", "where", "we're"), ("whose", "who's"), ("hear", "here"), ("buy", "bye"), ("weather", "whether"),
+    ("peace", "piece"), ("affect", "effect"), ("lose", "loose"), ("accept", "except"), ("advice", "advise"),
+    ("quiet", "quite"), ("through", "though", "thorough"), ("passed", "past"), ("principal", "principle"),
+    ("complement", "compliment"), ("brake", "break"), ("waist", "waste"), ("right", "write"), ("sight", "site", "cite"),
+    ("wear", "where"), ("led", "lead"), ("meet", "meat"), ("week", "weak"), ("wood", "would"), ("hole", "whole"),
+    ("male", "mail"), ("sale", "sell"), ("role", "roll"), ("aloud", "allowed"), ("board", "bored"), ("fair", "fare"),
+]
+TYPO_PAIRS_DATASET = "chirunder/github_typo_corrections"
+
+WORD_LIST_URL = "https://raw.githubusercontent.com/wolfgarbe/SymSpell/master/SymSpell.FrequencyDictionary/en-80k.txt"
+TOKEN = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+(?:[.,]\d+)*|[^\s\w]")
+SENTENCE_OPENERS = {".", "!", "?", ":", '"'}
+WORD = re.compile(r"^[a-z]+(?:'[a-z]+)?$")
+DIGITS = re.compile(r"^\d")
+
+
+def tokenize(text: str) -> list[str]:
+    """Lowercases words, except capitalized words mid-sentence, which become a single name token."""
+    tokens = []
+    for raw in TOKEN.findall(text.replace("’", "'")):
+        at_sentence_start = not tokens or tokens[-1] in SENTENCE_OPENERS
+        is_pronoun = raw == "I" or raw.startswith("I'")
+        tokens.append("<name>" if raw[0].isupper() and not at_sentence_start and not is_pronoun else raw.lower())
+    return tokens
+
+
+def fineweb_documents():
+    stream = load_dataset("HuggingFaceFW/fineweb-edu", split="train", streaming=True).shuffle(seed=7, buffer_size=20000)
+    for row in stream:
+        yield row["text"]
+
+
+def smoltalk_documents():
+    stream = load_dataset("HuggingFaceTB/smoltalk", "all", split="train", streaming=True).shuffle(seed=7, buffer_size=20000)
+    for row in stream:
+        yield "\n".join(message["content"] for message in row["messages"] if message["role"] != "system")
+
+
+def open_assistant_documents():
+    for row in load_dataset("OpenAssistant/oasst1", split="train"):
+        if row["lang"] == "en":
+            yield row["text"]
+
+
+def load_corpus(token_budgets: dict[str, int]) -> tuple[list[np.ndarray], dict[str, int]]:
+    sources = {"fineweb": fineweb_documents, "smoltalk": smoltalk_documents, "oasst": open_assistant_documents}
+    raw_ids = {"<s>": 0}
+    documents = []
+    for name, budget in token_budgets.items():
+        started, used = time.time(), 0
+        for text in sources[name]():
+            tokens = tokenize(text)
+            if len(tokens) < 8:
+                continue
+            documents.append(np.array([0] + [raw_ids.setdefault(token, len(raw_ids)) for token in tokens], dtype=np.int32))
+            used += len(tokens)
+            if used >= budget:
+                break
+        print(f"{name}: {used:,} tokens in {time.time() - started:.0f}s", flush=True)
+    return documents, raw_ids
+
+
+def build_vocabulary(documents: list[np.ndarray], raw_ids: dict[str, int], size: int) -> tuple[list[str], np.ndarray]:
+    with urllib.request.urlopen(WORD_LIST_URL) as response:
+        dictionary = {line.split()[0] for line in response.read().decode().splitlines() if line.strip()}
+    counts = np.bincount(np.concatenate(documents), minlength=len(raw_ids))
+    allowed = [(token, identifier) for token, identifier in raw_ids.items() if (WORD.match(token) and token in dictionary) or (len(token) == 1 and not token.isalnum())]
+    allowed = [pair for pair in allowed if pair[0] != "<s>"]
+    allowed.sort(key=lambda pair: (-counts[pair[1]], pair[0]))
+    vocabulary = SPECIAL_TOKENS + [token for token, _ in allowed[: size - len(SPECIAL_TOKENS)]]
+
+    token_to_id = {token: index for index, token in enumerate(vocabulary)}
+    lookup = np.full(len(raw_ids), UNK, dtype=np.int32)
+    for token, identifier in raw_ids.items():
+        if token == "<s>":
+            lookup[identifier] = START
+        elif token == "<name>":
+            lookup[identifier] = NAME
+        elif DIGITS.match(token):
+            lookup[identifier] = NUMBER
+        elif token in token_to_id:
+            lookup[identifier] = token_to_id[token]
+    return vocabulary, lookup
+
+
+def is_target_word(token: str) -> bool:
+    return len(token) >= 2 and token not in SPECIAL_TOKENS and WORD.match(token) is not None
+
+
+def within_one_edit(first: str, second: str) -> bool:
+    if first == second or abs(len(first) - len(second)) > 1:
+        return False
+    if len(first) == len(second):
+        differences = [index for index, (a, b) in enumerate(zip(first, second)) if a != b]
+        if len(differences) == 1:
+            return True
+        return len(differences) == 2 and differences[1] == differences[0] + 1 and first[differences[0]] == second[differences[1]] and first[differences[1]] == second[differences[0]]
+    shorter, longer = sorted((first, second), key=len)
+    return any(longer[:index] + longer[index + 1 :] == shorter for index in range(len(longer)))
+
+
+def strip_punctuation(token: str) -> str:
+    return token.strip("\"'()[]{}<>.,;:!?*_`#-=/\\")
+
+
+def real_mistakes_for_training(vocabulary: set[str]) -> Counter:
+    """Counts real typed-for-expected word swaps, using only the half of the dev split that eval doesn't tune on."""
+    counts = Counter()
+    for row in load_dataset(TYPO_PAIRS_DATASET, split="train"):
+        typed_tokens, fixed_tokens = (row["text"] or "").split(), (row["correction"] or "").split()
+        if len(typed_tokens) != len(fixed_tokens):
+            continue
+        different = [index for index, (first, second) in enumerate(zip(typed_tokens, fixed_tokens)) if first != second]
+        if len(different) != 1:
+            continue
+        typed, expected = strip_punctuation(typed_tokens[different[0]]), strip_punctuation(fixed_tokens[different[0]])
+        digest = int(hashlib.sha1(f"{typed}|{expected}".lower().encode()).hexdigest(), 16)
+        if digest % 2 != 0 or (digest >> 1) % 2 != 0:
+            continue
+        typed, expected = typed.lower(), expected.lower()
+        if typed != expected and len(typed) >= MIN_TYPED_LENGTH and typed in vocabulary and expected in vocabulary and is_target_word(typed) and is_target_word(expected):
+            counts[(typed, expected)] += 1
+    return Counter({pair: count for pair, count in counts.items() if count >= MIN_REAL_MISTAKE_COUNT})
+
+
+def build_confusions(vocabulary: list[str], use_real_mistakes: bool) -> dict[str, list[tuple[str, int]]]:
+    """Maps a typed word to words people really meant by it, most likely first."""
+    known = set(vocabulary)
+    counts = real_mistakes_for_training(known) if use_real_mistakes else Counter()
+    print(f"real mistake pairs={len(counts):,}", flush=True)
+    for group in HOMOPHONES:
+        for typed in group:
+            for expected in group:
+                if typed != expected and typed in known and expected in known and len(typed) >= MIN_TYPED_LENGTH:
+                    counts[(typed, expected)] += HOMOPHONE_WEIGHT
+    confusions = defaultdict(list)
+    for (typed, expected), count in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
+        confusions[typed].append((expected, count))
+    return {typed: options[:MAX_NEIGHBORS] for typed, options in confusions.items()}
+
+
+def build_real_mistake_tables(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> tuple[np.ndarray, np.ndarray]:
+    """For each intended word, lists the words typed by mistake and how often, for sampling."""
+    token_to_id = {token: index for index, token in enumerate(vocabulary)}
+    mistakes = defaultdict(list)
+    for typed, options in confusions.items():
+        for expected, count in options:
+            mistakes[token_to_id[expected]].append((token_to_id[typed], count))
+    typed_table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32)
+    weights = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.float32)
+    for expected, options in mistakes.items():
+        options = sorted(options, key=lambda option: -option[1])[:MAX_NEIGHBORS]
+        typed_table[expected, : len(options)] = [typed for typed, _ in options]
+        weights[expected, : len(options)] = [count**0.5 for _, count in options]
+    return typed_table, weights
+
+
+def build_neighbors(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> np.ndarray:
+    """Lists known mix-ups first, then the most common words one edit away, the same order the browser uses."""
+    token_to_id = {token: index for index, token in enumerate(vocabulary)}
+    buckets = defaultdict(set)
+    for index, token in enumerate(vocabulary):
+        if not is_target_word(token):
+            continue
+        buckets[token].add(index)
+        for position in range(len(token)):
+            buckets[token[:position] + token[position + 1 :]].add(index)
+
+    table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32)
+    for index, token in enumerate(vocabulary):
+        if not is_target_word(token):
+            continue
+        nearby = set(buckets[token])
+        for position in range(len(token)):
+            nearby |= buckets[token[:position] + token[position + 1 :]]
+        one_edit = sorted(other for other in nearby if within_one_edit(token, vocabulary[other]))
+        known_mixups = [token_to_id[expected] for expected, _ in confusions.get(token, [])]
+        found = list(dict.fromkeys(known_mixups + one_edit))
+        table[index, : min(len(found), MAX_NEIGHBORS)] = found[:MAX_NEIGHBORS]
+    return table
+
+
+class MatMulLinear(nn.Module):
+    """Linear layer that exports as MatMul so ONNX Runtime dynamic quantization applies."""
+
+    def __init__(self, inputs: int, outputs: int):
+        super().__init__()
+        self.weight = nn.Parameter(nn.init.xavier_uniform_(torch.empty(inputs, outputs)))
+        self.bias = nn.Parameter(torch.zeros(outputs))
+
+    def forward(self, values):
+        return values.matmul(self.weight) + self.bias
+
+
+class Gru(nn.Module):
+    """GRU built from MatMul layers because ONNX Runtime cannot quantize its GRU operator."""
+
+    def __init__(self, inputs: int, hidden: int):
+        super().__init__()
+        self.hidden = hidden
+        self.reset_gate = MatMulLinear(inputs + hidden, hidden)
+        self.update_gate = MatMulLinear(inputs + hidden, hidden)
+        self.proposal_input = MatMulLinear(inputs, hidden)
+        self.proposal_state = MatMulLinear(hidden, hidden)
+
+    def forward(self, sequence):
+        state = sequence[:, 0, :].sum(-1, keepdim=True).mul(0).repeat(1, self.hidden)
+        for step in range(sequence.shape[1]):
+            current = sequence[:, step, :]
+            joined = torch.cat([current, state], dim=-1)
+            reset = torch.sigmoid(self.reset_gate(joined))
+            update = torch.sigmoid(self.update_gate(joined))
+            proposal = torch.tanh(self.proposal_input(current) + reset * self.proposal_state(state))
+            state = (1 - update) * proposal + update * state
+        return state
+
+
+class WrongWordModel(nn.Module):
+    """Scores the typed word and its neighbors against the words on both sides."""
+
+    def __init__(self, vocabulary_size: int, embedding: int, hidden: int):
+        super().__init__()
+        self.embedding = nn.Embedding(vocabulary_size, embedding, padding_idx=PAD)
+        self.word_bias = nn.Embedding(vocabulary_size, 1, padding_idx=PAD)
+        self.left = Gru(embedding, hidden)
+        self.right = Gru(embedding, hidden)
+        self.mix = MatMulLinear(hidden * 2, hidden)
+        self.project = MatMulLinear(hidden, embedding)
+        self.typed_bonus = nn.Parameter(torch.tensor(1.0))
+        self.register_buffer("typed_position", F.one_hot(torch.tensor(0), CANDIDATES).float())
+
+    def forward(self, left, right, candidates):
+        context = torch.cat([self.left(self.embedding(left)), self.right(self.embedding(right))], dim=-1)
+        summary = self.project(torch.tanh(self.mix(context)))
+        scores = (self.embedding(candidates) * summary.unsqueeze(1)).sum(-1) + self.word_bias(candidates).squeeze(-1)
+        return scores + self.typed_bonus * self.typed_position
+
+
+class Sampler:
+    """Draws training cases by swapping real words for nearby real words."""
+
+    def __init__(self, tokens: torch.Tensor, vocabulary: list[str], neighbors: np.ndarray, real_typed: np.ndarray, real_weights: np.ndarray, device: str):
+        self.tokens = tokens
+        self.device = device
+        self.vocabulary = vocabulary
+        self.neighbor_table = neighbors
+        self.real_typed_table = real_typed
+        self.real_weights_table = real_weights
+        self.real_typed = torch.from_numpy(real_typed).to(device).long()
+        self.real_weights = torch.from_numpy(real_weights).to(device)
+        self.has_real = self.real_weights.sum(1) > 0
+        self.neighbors = torch.from_numpy(neighbors).to(device).long()
+        self.neighbor_counts = (self.neighbors != PAD).sum(1)
+        self.lengths = torch.tensor([len(token) for token in vocabulary], device=device)
+        self.is_target = torch.tensor([is_target_word(token) for token in vocabulary], device=device)
+
+    def batch(self, size: int, corrupt_rate: float):
+        positions = torch.randint(LEFT_WINDOW + 1, len(self.tokens) - RIGHT_WINDOW - 1, (size,), device=self.device)
+        target = self.tokens[positions]
+        counts = self.neighbor_counts[target]
+        choice = (torch.rand(size, device=self.device) * counts.clamp(min=1)).long()
+        corrupted = self.neighbors[target, choice]
+        weights = self.real_weights[target]
+        weights = weights.where(self.has_real[target].unsqueeze(1), torch.ones_like(weights))
+        real_choice = torch.multinomial(weights, 1).squeeze(1)
+        uses_real = self.has_real[target] & (torch.rand(size, device=self.device) < REAL_MISTAKE_SHARE)
+        corrupted = torch.where(uses_real, self.real_typed[target, real_choice], corrupted)
+        counts = counts.where(~uses_real, torch.ones_like(counts))
+        is_corrupted = (torch.rand(size, device=self.device) < corrupt_rate) & (counts > 0)
+        typed = torch.where(is_corrupted, corrupted, target)
+        candidates = torch.cat([typed.unsqueeze(1), self.neighbors[typed]], dim=1)
+        matches = candidates == target.unsqueeze(1)
+        valid = self.is_target[target] & (self.lengths[typed] >= MIN_TYPED_LENGTH) & (self.neighbor_counts[typed] > 0) & matches.any(1)
+
+        positions, candidates, matches, is_corrupted = positions[valid], candidates[valid], matches[valid], is_corrupted[valid]
+        count = len(positions)
+        left_offsets = torch.arange(-LEFT_WINDOW, 0, device=self.device)
+        left = self.tokens[positions.unsqueeze(1) + left_offsets]
+        right = self.tokens[positions.unsqueeze(1) + torch.arange(1, RIGHT_WINDOW + 1, device=self.device)]
+
+        visible_right = torch.randint(0, RIGHT_WINDOW + 1, (count,), device=self.device).unsqueeze(1)
+        right = right.where(torch.arange(RIGHT_WINDOW, device=self.device).unsqueeze(0) < visible_right, torch.tensor(PAD, device=self.device)).flip(1)
+
+        is_short = torch.rand(count, device=self.device) < 0.2
+        kept = torch.randint(0, LEFT_WINDOW, (count,), device=self.device).unsqueeze(1)
+        distance_from_end = (-left_offsets).unsqueeze(0)
+        hidden_left = is_short.unsqueeze(1) & (distance_from_end > kept + 1)
+        marks_start = is_short.unsqueeze(1) & (distance_from_end == kept + 1)
+        left = left.where(~hidden_left, torch.tensor(PAD, device=self.device)).where(~marks_start, torch.tensor(START, device=self.device))
+        return left, right, candidates, matches.long().argmax(1), is_corrupted
+
+
+def mask_empty(scores, candidates):
+    return scores.where(candidates != PAD, torch.tensor(-1e4, device=scores.device))
+
+
+@torch.no_grad()
+def evaluate(model, sampler, batches: int, batch_size: int) -> dict:
+    model.eval()
+    clean_margins, corrupted_margins, corrupted_right = [], [], []
+    for _ in range(batches):
+        left, right, candidates, label, is_corrupted = sampler.batch(batch_size, 0.5)
+        scores = mask_empty(model(left, right, candidates), candidates)
+        best = scores[:, 1:].max(1)
+        margin = best.values - scores[:, 0]
+        clean_margins.append(margin[~is_corrupted])
+        corrupted_margins.append(margin[is_corrupted])
+        corrupted_right.append(best.indices[is_corrupted] + 1 == label[is_corrupted])
+    model.train()
+    clean = torch.cat(clean_margins)
+    margin = torch.cat(corrupted_margins)
+    right = torch.cat(corrupted_right)
+    results = {"clean_cases": len(clean), "corrupted_cases": len(margin)}
+    for false_rate in (0.005, 0.001, 0.0005):
+        threshold = torch.quantile(clean[:200000].float(), 1 - false_rate).item()
+        results[f"caught_at_false_{false_rate}"] = round(((margin >= threshold) & right).float().mean().item(), 4)
+        results[f"threshold_at_false_{false_rate}"] = round(threshold, 2)
+    return results
+
+
+def export_onnx(model: WrongWordModel, output: Path, sampler: Sampler) -> dict:
+    model = model.cpu().eval()
+    float_path = output / "model.onnx"
+    example = (torch.ones(2, LEFT_WINDOW, dtype=torch.long), torch.ones(2, RIGHT_WINDOW, dtype=torch.long), torch.ones(2, CANDIDATES, dtype=torch.long))
+    torch.onnx.export(
+        model,
+        example,
+        float_path,
+        input_names=["left", "right", "candidates"],
+        output_names=["scores"],
+        dynamic_axes={"left": {0: "batch"}, "right": {0: "batch"}, "candidates": {0: "batch"}, "scores": {0: "batch"}},
+        opset_version=17,
+        dynamo=False,
+    )
+    quantized = {
+        "model_quint8.onnx": ["MatMul", "Gather"],
+        "model_matmul_quint8.onnx": ["MatMul"],
+    }
+    for name, operators in quantized.items():
+        quantize_dynamic(float_path, output / name, weight_type=QuantType.QUInt8, op_types_to_quantize=operators)
+
+    sampler_cpu = Sampler(sampler.tokens.cpu(), sampler.vocabulary, sampler.neighbor_table, sampler.real_typed_table, sampler.real_weights_table, "cpu")
+    left, right, candidates, label, _ = sampler_cpu.batch(4096, 0.5)
+    inputs = {"left": left.numpy().astype(np.int64), "right": right.numpy().astype(np.int64), "candidates": candidates.numpy().astype(np.int64)}
+    with torch.no_grad():
+        reference = model(left, right, candidates).numpy()
+
+    def run(path: Path) -> np.ndarray:
+        return ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]).run(None, inputs)[0]
+
+    def margins(scores: np.ndarray) -> np.ndarray:
+        scores = np.where(inputs["candidates"] != PAD, scores, -1e4)
+        return scores[:, 1:].max(1) - scores[:, 0]
+
+    report = {"float_vs_torch_max_difference": float(np.abs(run(float_path) - reference).max())}
+    report["float_mb"] = round(float_path.stat().st_size / 1e6, 2)
+    for name in quantized:
+        scores = run(output / name)
+        report[name] = {
+            "mb": round((output / name).stat().st_size / 1e6, 2),
+            "pick_agreement": round(float((scores.argmax(1) == reference.argmax(1)).mean()), 4),
+            "mean_margin_difference": round(float(np.abs(margins(scores) - margins(reference)).mean()), 3),
+        }
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo", default="cfahlgren1/tinyfix-realword")
+    parser.add_argument("--run-name", default="v4")
+    parser.add_argument("--real-mistakes", action="store_true", help="Also train on GitHub word swaps. Scored worse than homophones alone in v3.")
+    parser.add_argument("--fineweb-tokens", type=int, default=70_000_000)
+    parser.add_argument("--smoltalk-tokens", type=int, default=50_000_000)
+    parser.add_argument("--vocabulary", type=int, default=20_000)
+    parser.add_argument("--embedding", type=int, default=64)
+    parser.add_argument("--hidden", type=int, default=256)
+    parser.add_argument("--steps", type=int, default=60_000)
+    parser.add_argument("--batch", type=int, default=4096)
+    parser.add_argument("--learning-rate", type=float, default=2e-3)
+    parser.add_argument("--output", default="/tmp/tinyfix-realword")
+    parser.add_argument("--no-upload", action="store_true")
+    args = parser.parse_args()
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    output = Path(args.output)
+    output.mkdir(parents=True, exist_ok=True)
+    print(f"device={device} args={vars(args)}", flush=True)
+
+    documents, raw_ids = load_corpus({"fineweb": args.fineweb_tokens, "smoltalk": args.smoltalk_tokens, "oasst": 10**12})
+    vocabulary, lookup = build_vocabulary(documents, raw_ids, args.vocabulary)
+    print(f"documents={len(documents):,} raw_types={len(raw_ids):,} vocabulary={len(vocabulary):,}", flush=True)
+
+    order = np.random.default_rng(7).permutation(len(documents))
+    held_out = max(1, len(documents) // 20)
+    train_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[held_out:]])]).to(device).long()
+    validation_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[:held_out]])]).to(device).long()
+    print(f"train={len(train_tokens):,} validation={len(validation_tokens):,} unknown={(train_tokens == UNK).float().mean().item():.3f}", flush=True)
+    del documents
+
+    confusions = build_confusions(vocabulary, args.real_mistakes)
+    (output / "confusions.txt").write_text("".join(f"{typed} {' '.join(expected for expected, _ in options)}\n" for typed, options in confusions.items()))
+    neighbors = build_neighbors(vocabulary, confusions)
+    real_typed, real_weights = build_real_mistake_tables(vocabulary, confusions)
+    print(f"words with neighbors={(neighbors[:, 0] != PAD).sum():,}", flush=True)
+    (output / "vocab.txt").write_text("\n".join(vocabulary) + "\n")
+
+    train_sampler = Sampler(train_tokens, vocabulary, neighbors, real_typed, real_weights, device)
+    validation_sampler = Sampler(validation_tokens, vocabulary, neighbors, real_typed, real_weights, device)
+
+    model = WrongWordModel(len(vocabulary), args.embedding, args.hidden).to(device)
+    parameters = sum(parameter.numel() for parameter in model.parameters())
+    print(f"parameters={parameters:,}", flush=True)
+    optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=1e-6)
+    schedule = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.learning_rate, total_steps=args.steps, pct_start=0.1)
+
+    started = time.time()
+    running_loss = 0.0
+    report_every = max(args.steps // 40, 1)
+    for step in range(1, args.steps + 1):
+        left, right, candidates, label, _ = train_sampler.batch(args.batch, 0.5)
+        loss = F.cross_entropy(mask_empty(model(left, right, candidates), candidates), label)
+        optimizer.zero_grad(set_to_none=True)
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+        optimizer.step()
+        schedule.step()
+        running_loss += loss.item()
+        if step % report_every == 0:
+            print(f"step {step}/{args.steps} loss={running_loss / report_every:.4f} elapsed={time.time() - started:.0f}s", flush=True)
+            running_loss = 0.0
+        if step % (args.steps // 5 or 1) == 0:
+            print(f"validation step {step}: {evaluate(model, validation_sampler, 60, args.batch)}", flush=True)
+
+    validation = evaluate(model, validation_sampler, 200, args.batch)
+    print(f"final validation: {validation}", flush=True)
+    export_report = export_onnx(model, output, validation_sampler)
+    print(f"export: {export_report}", flush=True)
+
+    metrics = {
+        "args": vars(args),
+        "parameters": parameters,
+        "validation": validation,
+        "export": export_report,
+        "train_tokens": len(train_tokens),
+        "train_minutes": round((time.time() - started) / 60, 1),
+    }
+    (output / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
+    print(json.dumps(metrics, indent=2), flush=True)
+
+    if args.no_upload:
+        print(f"Skipping upload; artifacts in {output}", flush=True)
+        return
+    api = HfApi(token=os.environ["HF_TOKEN"])
+    api.create_repo(args.repo, repo_type="model", private=True, exist_ok=True)
+    api.upload_folder(folder_path=str(output), repo_id=args.repo, path_in_repo=args.run_name, repo_type="model")
+    print(f"Uploaded to {args.repo}/{args.run_name}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

v1/metrics.json

@@ -1,43 +0,0 @@
-{
-  "args": {
-    "repo": "cfahlgren1/tinyfix-realword",
-    "run_name": "v1",
-    "fineweb_tokens": 70000000,
-    "smoltalk_tokens": 50000000,
-    "vocabulary": 20000,
-    "embedding": 64,
-    "hidden": 256,
-    "steps": 60000,
-    "batch": 4096,
-    "learning_rate": 0.002,
-    "output": "/tmp/tinyfix-realword",
-    "no_upload": false
-  },
-  "parameters": 1941345,
-  "validation": {
-    "clean_cases": 210578,
-    "corrupted_cases": 210034,
-    "caught_at_false_0.005": 0.7136,
-    "threshold_at_false_0.005": 3.13,
-    "caught_at_false_0.001": 0.5621,
-    "threshold_at_false_0.001": 5.13,
-    "caught_at_false_0.0005": 0.4977,
-    "threshold_at_false_0.0005": 5.93
-  },
-  "export": {
-    "float_vs_torch_max_difference": 5.7220458984375e-06,
-    "float_mb": 7.81,
-    "model_quint8.onnx": {
-      "mb": 2.09,
-      "pick_agreement": 0.9986,
-      "mean_margin_difference": 0.104
-    },
-    "model_matmul_quint8.onnx": {
-      "mb": 5.99,
-      "pick_agreement": 0.9967,
-      "mean_margin_difference": 0.092
-    }
-  },
-  "train_tokens": 118291955,
-  "train_minutes": 24.9
-}

v1/model.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:57f94203c18a32e6e0585ad0c7f8cfea4daf4fff175b8dd91afdc2f987034bd3
-size 7814671

v1/model_matmul_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1b35513a64081c59bd612cb9088a3d4dadcb63bc4dbfd3075ceac6e0259ef454
-size 5991703

v1/model_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:563d98809074312be6ed156c516826c3f4b9a69764c5319c0aecc64dfbfdfb05
-size 2092690

v2/metrics.json

@@ -1,43 +0,0 @@
-{
-  "args": {
-    "repo": "cfahlgren1/tinyfix-realword",
-    "run_name": "v2",
-    "fineweb_tokens": 70000000,
-    "smoltalk_tokens": 50000000,
-    "vocabulary": 20000,
-    "embedding": 64,
-    "hidden": 256,
-    "steps": 60000,
-    "batch": 4096,
-    "learning_rate": 0.002,
-    "output": "/tmp/tinyfix-realword",
-    "no_upload": false
-  },
-  "parameters": 1941345,
-  "validation": {
-    "clean_cases": 200005,
-    "corrupted_cases": 201620,
-    "caught_at_false_0.005": 0.7301,
-    "threshold_at_false_0.005": 3.1,
-    "caught_at_false_0.001": 0.5732,
-    "threshold_at_false_0.001": 5.25,
-    "caught_at_false_0.0005": 0.4914,
-    "threshold_at_false_0.0005": 6.26
-  },
-  "export": {
-    "float_vs_torch_max_difference": 5.7220458984375e-06,
-    "float_mb": 7.81,
-    "model_quint8.onnx": {
-      "mb": 2.09,
-      "pick_agreement": 0.998,
-      "mean_margin_difference": 0.091
-    },
-    "model_matmul_quint8.onnx": {
-      "mb": 5.99,
-      "pick_agreement": 0.997,
-      "mean_margin_difference": 0.081
-    }
-  },
-  "train_tokens": 118291713,
-  "train_minutes": 24.9
-}

v2/model.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:45a3d5d676a972801b6f9bd51d0f2e6439278694eace52aadb6a37725d7e3a51
-size 7814671

v2/model_matmul_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7c7ba1c17231fb295c5c1b36b451f3e14d46c30bde139e9703d5115e475e4a95
-size 5991706

v3/confusions.txt

@@ -1,882 +0,0 @@
-it's its is their all invoke
-you your who if yo
-are is or and area at array of
-this these if that is thus
-return returns raise render output
-send sent sending
-you're your
-not note
-use used you
-simple simply
-weather whether
-your you're multiple our
-they the there you
-url file protocol
-filed field
-were where we're was
-test tests
-with will without within using about this we while
-know known
-off of error
-ran run
-create created creating update
-instance instances instant
-affect effect
-save safe same
-file files filed object
-expect except expected
-method methods message class
-provide provides providing
-value values null
-left right top
-change changed changing
-enable enabled
-has as must
-objects object
-using use uses with
-event even
-one once of first two
-through though thorough
-work works worked worker
-sent send
-advise advice
-built build
-instruction instructions
-make may means
-their they're there
-int in string into double
-javascript html object
-received receive
-right write left bottom top
-two too to
-weak week
-build builds
-ever every
-filter filters
-for of form four more are fork if
-now know new and
-values value
-what that which this why
-accept except
-advice advise
-allowed aloud
-aloud allowed
-board bored
-bored board
-brake break
-break brake
-cite sight site
-complement compliment
-compliment complement
-container containers contains pod
-effect affect
-except accept expect
-fair fare
-fare fair far
-hear here
-here hear
-hole whole
-its it's the
-lead led leaf
-led lead
-loose lose
-lose loose
-mail male
-male mail
-meat meet
-meet meat
-passed past
-past passed
-peace piece
-piece peace
-principal principle
-principle principal
-quiet quite
-quite quiet
-role roll
-roll role
-sale sell
-sell sale
-sight cite site
-site cite sight
-starts start starting
-than then
-then than
-there their they're they there's
-they're their there
-thorough though through
-though thorough through
-too two top
-waist waste
-waste waist
-we're were where
-wear where
-week weak
-where we're wear were
-whether weather
-who's whose
-whole hole
-whose who's
-wood would
-would wood could
-write right written writing
-changes change changed
-creates create
-generate generated generates remove
-mean means
-others other
-remove remote removes removing
-text test binary paragraph background
-uses used
-contain contains contained
-first last
-provided provide provides
-run runs ran
-ans and
-based base
-case cases cast
-determine determines determined
-ensure ensures
-extensions types policy
-get gets git getting be
-mayor major
-provides provide
-response request
-staring starting
-tag tags
-county count
-generated generate
-minimum maximum
-post port page
-reminder remainder
-director directory
-evaluate enable evaluated
-receive receives received
-that this of pace that's who
-after before by matched
-maybe may
-metrics metric
-mounting updating
-potential potentially
-show shown
-which with who
-above below
-dat data day
-find finds
-interface interfaces
-mount amount
-solution solutions
-stored store stores
-those these
-user used author
-controlled uncontrolled
-difference different differences
-err error
-image images imagine input
-request response
-suit suite
-welcome welcomed
-basics basic
-commands command
-constrain constraint
-dedicate dedicated
-driver drivers
-expression expressions condition
-generates generate
-last least latest
-module modules foo
-node nodes
-parent child
-peak peek
-power powerful
-specially especially
-sting string
-the its these for if of is their by
-var let
-consist consists
-covert convert
-dose does
-download install downloaded downloading
-fails fail
-flatten flattened
-frequent frequency
-gets get
-like look luck line
-looks look
-manor manner
-modify modifying create modified
-session sessions
-sockets tunnels
-target targets
-them the they
-tow two
-users uses
-variable variables option
-worse worst
-app apps
-cab can
-clients client
-creating created
-indicates indicate
-match matches march
-parse parser parsed
-produces produced producers
-property properly properties proper
-provider provided
-red read
-result results data
-slot slots
-strings string
-again against
-blow below
-buy by
-can cannot and be could
-cant can want
-class classes attribute module clash instance
-collection collections
-cpu gpu
-decorated decorate
-defined defines
-directly directory
-four five for
-functional function
-identify identity
-input inputs
-look looks
-manager manage
-new now the
-photo ad
-provision provisioning
-start starting started starter
-string strings strict number
-transforms transform
-tread thread
-versions version
-below belong
-close closed clone
-come comes some
-computes computed
-defines defined
-display displays displaying
-does is
-enabled disabled
-entire complete
-expressions expression
-inputs input outputs
-interesting interested
-key keys prefix value
-met me
-null false
-producer consumer
-record records
-represents represent
-row column span
-set get
-temporary temporarily
-thee the
-transform transforms
-translation translations
-very every
-and then for that given
-arm check
-array slice
-belongs belong
-button sensor
-commons common
-configuration configuring configure
-could would cannot
-div column li min
-document documentation
-easy easily
-fill will
-follows follow
-hat that
-hen when
-hold holds held
-how what
-innovation education
-ins is
-issued issues
-latter later first
-making make
-mix mixed
-more or
-part party
-paths files
-phase job
-que queue
-register registers
-returned return
-running run
-second third section
-sections section
-ten five
-theme them
-tutorial tutorials
-want what
-worker master web
-added add removed
-applies applied
-begging beginning
-being been
-black block
-both bother what
-bumps bump
-comment comments quote comet
-definition definitions
-deploying deploy
-fixed fixes variable closed
-following follow
-got get have
-handles handle
-happen happens
-identity identify
-import imports line
-install deploy build
-isn't aren't
-item items time
-label labels heading
-library libraries
-ones once one
-outputs output
-particular particularly
-passes pass
-password service
-patterns pattern
-projects project
-reference references refer
-rest reset
-side size
-sources source
-supported support possible
-term terms turn
-title pages body
-tokens token circles
-ton to
-when with where
-account accounts
-actually actual
-adapter adapted
-afterwards afterward
-all allow most the
-answer answers
-any ant one
-appearing appear
-assignments assignment
-author name
-backed baked
-became become
-calling called
-charts lines
-child children
-commit commits
-computation computations
-condition conditions
-consumer producer
-contained containing
-controls control
-convention conventional
-crates creates
-credit credits
-decided decide
-descriptions description
-dist pd
-distribute distributed
-don't do
-dot do
-easily easy
-elliptical elliptic
-epoch epochs
-extension extensions
-fir for
-flat flag
-float string
-format value
-fourth fifth
-happens happen
-hard soft
-having have
-hero player
-inherits inherited
-integrate integrates
-integrated integrate integrating
-introduces introduced
-involved involve
-issue issued
-just you
-large larger
-leak leaky
-leave leaf
-lightning lighting
-live life online
-logging monitoring
-manage manager
-model module mode
-name type description node
-notification notifications
-occurs occur
-offer offers
-parts part
-pass passes
-perform performed
-point pointer
-prepare prepared
-query mutation
-readers writers
-records record
-releases release
-removed remove
-reporting build
-selects select
-server application
-service file
-shot shoot
-size length
-specify specifies
-system snippets
-take taken
-terminate list resume
-thanks thank
-these which
-training train
-transactions transaction
-update upgrade
-validate valid
-verify validate
-vertices vertical
-wants want
-was is has
-width with
-xml java
-about amount
-adapting adopting
-allocate allocated
-analyses analyzes
-android ios
-approached approach
-aren't isn't
-assembly assemble
-await return
-axis axes
-back forward
-backup restore
-backwards backward
-bundles bundle
-caching cache
-calculates calculate
-calculation calculations
-cane can
-cause because
-colors color
-comments comment
-compatible compatibility
-compliments complements
-computer computers
-consequently sequentially
-containing contain contains
-contracts contract
-contributors contributor
-convert check conversion
-deliver delivery
-delivery automate
-demonstrates demonstrate
-dependencies steps modules
-detect detects
-did didn't
-discuss discussed
-docker singularity
-double float
-drop add
-dump dumb
-east each
-enrollment authentication
-entry property
-error exception
-executed execute executes
-exiting exciting
-explain explained
-explicit explicitly
-facilitate facilitates
-faction factor
-fail fails
-foo bar
-formats relations
-forward backward forwards
-generation generating
-gives give
-granting grant
-handful helpful
-handle handled
-handlers handles
-hash hashed hashes
-haven have
-header headers
-hers her
-incoming outgoing
-index behavior indexed
-initiatives initiates
-issues issued
-iterator iterate
-japan japanese
-job jobs
-less loss
-list least dict set
-load create
-locale local
-mine mind
-models model
-much many
-notifications notification
-number minute
-open lock
-oracle mysql
-participants participant
-permissions permission
-php apache
-pods pod
-process processor
-purposes purpose
-question questions
-rails backup
-raises raised
-relations relationships
-remainder reminder
-repository repositories
-retrieve retrieved retrieving remove
-returning return
-saves save
-score loss
-secrete secret
-seems seem
-sequence sequences
-sin in
-since sense
-slave replica
-statement query
-sue sure
-suppose supposed
-supposed supported
-sync window
-tailed tailored
-taken take
-task future
-thank thanks
-themes theme
-there's there find
-timer time
-top topic
-trigger size
-turns turn
-typical typically
-underline underlying
-understand understanding understands
-unsigned signed
-val unit
-valued values
-vector matrix
-wallet wallets hosting
-wand want and
-what's what
-while will
-will should
-words word works
-access read
-accounts account
-action msg
-actions sections
-activity activities
-add at
-address error
-admin instructor
-adopt adapt
-agents agent
-aggregate aggregated
-aim aims
-alternative alternatively
-anchor guideline
-angles directions
-apart part
-application container
-apply link
-aria area
-arrive arrived
-art are
-attempt attempting
-attributes arguments
-automatic automatically
-bank blank
-bash sh
-basic destroy
-batch batches
-batteries services
-become became
-believe belief
-between and
-big bit
-branches values
-bread breed
-breaking braking
-buffer buffers vector window
-bus bugs
-but where
-cable view
-cache mail
-calculating calculate
-card item
-cards form
-channel file guild
-chart operator
-check integration
-checked health
-checkout check
-checkpoint checkpoints
-chose chooses
-clicked click
-clone cloned
-cluster topic
-coat color
-collect collects
-comparable akin
-compute observe
-connections connection
-construction constructing
-contract contrast
-contribute contributed
-convenience inconvenience
-coping copying
-copy copied
-crashes crashed
-creation deletion
-custom customize
-cut copy create
-cycling cyclic
-dead died
-debug display log
-default black
-del de
-described describe
-designate designated
-detector detection
-diamond oval
-directed directly
-direction direct directory
-directions directories
-doe do
-down done
-dumping inspecting
-elements method
-end stop
-ended ends
-engine bind
-equations equation
-essential essentially
-exceptional exceptionally
-extensive extension
-false stop
-featured features
-fifth sixth
-fin find
-finding find
-finish release
-followed subscribe
-forgot forget
-form forum
-fragments fragment
-gateway gateways
-gather cache
-gem get
-getting obtaining
-gpu cpu
-graduate graduates
-graph session
-gray grey
-great greater
-guide guides
-have behave half
-hidden hide
-high highly
-host domain
-idea ideas
-ignites ignite
-implementing of
-imported import
-including includes
-incorporate incorporation
-indices indexes
-info into
-inheritance embedding
-installer install
-installing installation
-interactions interaction
-internal interval
-into from
-introduce import introduces
-introduced introduces
-inversions inversion
-iphone ios
-isolate isolated
-jan june
-john jane
-journal server
-junk chunks
-larger smaller
-layer tile
-legally formally
-light dark
-link datasets
-listed listen
-listen listening listens
-listener listeners
-listing listening
-lists listens
-loaded loader
-log logic
-long longer
-lookup looking
-loud cloud
-mainly may
-marker markup
-master latest
-matched match
-material geometry
-may can
-meeting meetings
-merchants merchant
-message massage
-middle toggle
-migration migrations
-minimal bash
-minute second
-mirrors mirror
-missing method
-monitoring monitor
-must should
-need have
-needed needing
-neg sum
-nil null
-nodules modules
-none non
-normal light
-notes detail
-occurrence occurrences
-offset duration
-old go
-omitted emitted
-only using
-opens open
-operation operator
-option operation
-orbit alert
-other or
-our own
-outlines outlined
-output download
-packaged native
-packages packaged
-palette pal
-panel plane
-pas par
-performing perform
-place placement
-policy policies runtime
-positive negative
-primitive primitives
-procedure procedures
-profile profiles
-program programs
-project module
-properties events
-proxies proxy
-python net
-quickly soon
-quotient remainder
-rates rate
-rations ratios
-rebuilt rebuild
-recognition it
-remember remembered
-removes replaces
-representation representations
-requirement requirements
-res research
-rescue rescued
-research resource
-residence resident
-roles role
-ruby python
-rule role
-same name safe
-sample custom sampled
-saved save
-say stay
-select selected
-selection selected
-sensitivity speed
-sensor operator
-separate single
-setting getting set
-shape shapes
-shared sharing
-shortest longest
-signal signals
-signals signal
-similarly similar
-simply just
-snapshot abs
-some more
-son so
-sort soft
-source destination
-spans span
-specific specified
-sports supports
-started necessary
-stat state
-state stated states
-stone code
-streams feeds
-style italics still
-subsets subset
-support suppose
-supporter supported
-synchronous asynchronous
-table schema tab
-testing caching
-tests testes
-thrown throw throws
-thy they
-time run timestamps
-toggle service
-tool id
-transition navigate transitions
-tree trees
-triggers triggered
-true debug
-turn turned
-typing type
-under in
-underlined underlying
-unknown na
-until in
-updated updates
-uploads files upload
-valid validate
-variant read
-version info release revision
-view few viewed
-visible visibility
-war directory
-watches watch
-way away method want
-whenever wherever
-why what which
-widows windows
-workspace volume
-world life
-wrap wraps
-writer reader
-writing serving
-yard yarn
-year years
-you'll you're

v3/metrics.json

@@ -1,43 +0,0 @@
-{
-  "args": {
-    "repo": "cfahlgren1/tinyfix-realword",
-    "run_name": "v3",
-    "fineweb_tokens": 70000000,
-    "smoltalk_tokens": 50000000,
-    "vocabulary": 20000,
-    "embedding": 64,
-    "hidden": 256,
-    "steps": 60000,
-    "batch": 4096,
-    "learning_rate": 0.002,
-    "output": "/tmp/tinyfix-realword",
-    "no_upload": false
-  },
-  "parameters": 1941345,
-  "validation": {
-    "clean_cases": 203358,
-    "corrupted_cases": 212256,
-    "caught_at_false_0.005": 0.6522,
-    "threshold_at_false_0.005": 3.19,
-    "caught_at_false_0.001": 0.4989,
-    "threshold_at_false_0.001": 5.06,
-    "caught_at_false_0.0005": 0.4239,
-    "threshold_at_false_0.0005": 6.01
-  },
-  "export": {
-    "float_vs_torch_max_difference": 6.198883056640625e-06,
-    "float_mb": 7.81,
-    "model_quint8.onnx": {
-      "mb": 2.09,
-      "pick_agreement": 0.9947,
-      "mean_margin_difference": 0.099
-    },
-    "model_matmul_quint8.onnx": {
-      "mb": 5.99,
-      "pick_agreement": 0.9932,
-      "mean_margin_difference": 0.082
-    }
-  },
-  "train_tokens": 118291713,
-  "train_minutes": 27.6
-}

v3/model.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:345a7272ad63a1ab801c7cd77a674004c52e1c81839b80714cd157abae223007
-size 7814671

v3/model_matmul_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:33dfd6c147803965f660e19da23b16f8b9a1e6678221a16ee3d2fa687cbc8561
-size 5991706

v3/model_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:17e2a33016de1f8aa503a466670c9edc8f78d83b2996f68bdbec7e8b082fa9d0
-size 2092693

v4/metrics.json

@@ -1,44 +0,0 @@
-{
-  "args": {
-    "repo": "cfahlgren1/tinyfix-realword",
-    "run_name": "v4",
-    "real_mistakes": false,
-    "fineweb_tokens": 70000000,
-    "smoltalk_tokens": 50000000,
-    "vocabulary": 20000,
-    "embedding": 96,
-    "hidden": 512,
-    "steps": 90000,
-    "batch": 4096,
-    "learning_rate": 0.002,
-    "output": "/tmp/tinyfix-realword",
-    "no_upload": false
-  },
-  "parameters": 4385921,
-  "validation": {
-    "clean_cases": 199439,
-    "corrupted_cases": 202365,
-    "caught_at_false_0.005": 0.7682,
-    "threshold_at_false_0.005": 3.08,
-    "caught_at_false_0.001": 0.6164,
-    "threshold_at_false_0.001": 5.39,
-    "caught_at_false_0.0005": 0.5517,
-    "threshold_at_false_0.0005": 6.25
-  },
-  "export": {
-    "float_vs_torch_max_difference": 1.71661376953125e-05,
-    "float_mb": 17.59,
-    "model_quint8.onnx": {
-      "mb": 4.54,
-      "pick_agreement": 0.9931,
-      "mean_margin_difference": 0.223
-    },
-    "model_matmul_quint8.onnx": {
-      "mb": 10.36,
-      "pick_agreement": 0.9926,
-      "mean_margin_difference": 0.215
-    }
-  },
-  "train_tokens": 118291713,
-  "train_minutes": 55.7
-}

v4/model.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3240885f048049bcce599956b9ef42d5f77a242f18284bb90ec5308967faa940
-size 17592977

v4/model_matmul_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b391462ec853f0e4714aa913f42f42cebc735ac4e927715a4e300404bf3a36f2
-size 10363288

v4/model_quint8.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:05802e12d893e7e3d2246ad6e1ad2b0bebadc29754e4cf7771c69c28e1f5c242
-size 4544276

v4/confusions.txt → wrong-word-confusions.txt

@@ -1,73 +1,74 @@
-accept except
-advice advise
-advise advice
-affect effect
-allowed aloud
-aloud allowed
-board bored
-bored board
-brake break
-break brake
-cite sight site
-complement compliment
-compliment complement
-effect affect
-except accept
-fair fare
-fare fair
+their there they're
+there their they're
+they're their there
+your you're
+you're your
+its it's
+it's its
+too two
+two too
+then than
+than then
+were where we're
+where were we're
+we're were where
+whose who's
+who's whose
 hear here
 here hear
-hole whole
-it's its
-its it's
-lead led
-led lead
-loose lose
+weather whether
+whether weather
+peace piece
+piece peace
+affect effect
+effect affect
 lose loose
-mail male
-male mail
-meat meet
-meet meat
+loose lose
+accept except
+except accept
+advice advise
+advise advice
+quiet quite
+quite quiet
+through though thorough
+though through thorough
+thorough through though
 passed past
 past passed
-peace piece
-piece peace
 principal principle
 principle principal
-quiet quite
-quite quiet
-right write
-role roll
-roll role
-sale sell
-sell sale
-sight cite site
-site cite sight
-than then
-their there they're
-then than
-there their they're
-they're their there
-thorough though through
-though thorough through
-through thorough though
-too two
-two too
+complement compliment
+compliment complement
+brake break
+break brake
 waist waste
 waste waist
-we're were where
-weak week
+right write
+write right
+sight site cite
+site sight cite
+cite sight site
 wear where
-weather whether
+where wear
+led lead
+lead led
+meet meat
+meat meet
 week weak
-were we're where
-where we're wear were
-whether weather
-who's whose
-whole hole
-whose who's
+weak week
 wood would
 would wood
-write right
-you're your
-your you're
+hole whole
+whole hole
+male mail
+mail male
+sale sell
+sell sale
+role roll
+roll role
+aloud allowed
+allowed aloud
+board bored
+bored board
+fair fare
+fare fair