v2: 13 MB browser build (int8 download, float compute), export/compress scripts, onnx evals

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 hero.png filter=lfs diff=lfs merge=lfs -text
+v2/web/weights.q8 filter=lfs diff=lfs merge=lfs -text

v2/compress.py

@@ -0,0 +1,120 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = ["onnx>=1.17", "numpy<3", "onnxruntime>=1.20"]
+# ///
+"""Shrinks a float ONNX model for download without changing how it runs.
+
+The browser computes in float32 (onnxruntime-web's int8 kernels are several times slower than its float ones), so
+only the download is compressed: every large weight is stored as int8 with a float32 scale per block of 64 values.
+The page expands them back to float32 and hands them to onnxruntime-web as the model's external data.
+
+Writes, next to each other:
+  graph.onnx      the model graph; large weights point into "weights.f32" (external data)
+  weights.q8      int8 values, then float32 scales, tensor by tensor
+  weights.json    for each tensor: where its int8 values and scales start, its shape, and where it goes in weights.f32
+
+  uv run compress.py v2-release/browser/model.onnx v2-release/web
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+import onnx
+from onnx import numpy_helper
+from onnx.external_data_helper import set_external_data
+
+BLOCK = 64
+MIN_ELEMENTS = 4096  # smaller tensors (norm weights) stay inside the graph as float32
+EXTERNAL = "weights.f32"
+
+
+def quantize(values: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    """Symmetric int8 with one scale per BLOCK consecutive values."""
+    flat = values.astype(np.float32).ravel()
+    padded = np.pad(flat, (0, (-len(flat)) % BLOCK)).reshape(-1, BLOCK)
+    scales = np.abs(padded).max(axis=1) / 127
+    scales[scales == 0] = 1
+    q = np.clip(np.round(padded / scales[:, None]), -127, 127).astype(np.int8)
+    return q.ravel()[: len(flat)], scales.astype(np.float32)
+
+
+def dequantize(q: np.ndarray, scales: np.ndarray, count: int) -> np.ndarray:
+    padded = np.pad(q.astype(np.float32), (0, (-count) % BLOCK)).reshape(-1, BLOCK)
+    return (padded * scales[:, None]).ravel()[:count]
+
+
+def dedupe_transposes(model) -> None:
+    """The output layer reuses the embedding table, but export stores a transposed copy. Keep one and transpose it
+    in the graph instead, which saves about 2.6 MB of download."""
+    from onnx import helper
+
+    graph = model.graph
+    arrays = {t.name: numpy_helper.to_array(t) for t in graph.initializer if t.dims and np.prod(t.dims) >= MIN_ELEMENTS}
+    for name, values in list(arrays.items()):
+        if values.ndim != 2:
+            continue
+        for other, candidate in arrays.items():
+            if other != name and candidate.shape == values.T.shape and np.array_equal(candidate, values.T):
+                transposed = f"{other}_transposed"
+                graph.node.insert(0, helper.make_node("Transpose", [other], [transposed], perm=[1, 0], name=transposed))
+                for node in graph.node:
+                    node.input[:] = [transposed if i == name else i for i in node.input]
+                graph.initializer.remove(next(t for t in graph.initializer if t.name == name))
+                print(f"stored {name} as a transpose of {other}")
+                return
+
+
+def compress(src: Path, out: Path) -> dict:
+    out.mkdir(parents=True, exist_ok=True)
+    model = onnx.load(str(src))
+    dedupe_transposes(model)
+    manifest, q8 = [], bytearray()
+    f32_offset = 0
+    for tensor in model.graph.initializer:
+        values = numpy_helper.to_array(tensor)
+        if values.dtype != np.float32 or values.size < MIN_ELEMENTS:
+            continue
+        q, scales = quantize(values)
+        entry = {"name": tensor.name, "shape": list(values.shape), "count": int(values.size), "f32_offset": f32_offset,
+                 "q_offset": len(q8), "scale_offset": len(q8) + len(q) + (-len(q)) % 4}
+        q8 += q.tobytes()
+        q8 += b"\0" * ((-len(q8)) % 4)  # keep scales 4-byte aligned for Float32Array views
+        q8 += scales.tobytes()
+        manifest.append(entry)
+        # Point the graph at weights.f32, which the browser builds by expanding weights.q8.
+        if not tensor.HasField("raw_data"):
+            tensor.raw_data = values.astype(np.float32).tobytes()
+            tensor.ClearField("float_data")
+        set_external_data(tensor, location=EXTERNAL, offset=f32_offset, length=values.size * 4)
+        tensor.ClearField("raw_data")
+        tensor.data_location = onnx.TensorProto.EXTERNAL
+        f32_offset += values.size * 4
+    onnx.save_model(model, str(out / "graph.onnx"))
+    (out / "weights.q8").write_bytes(bytes(q8))
+    (out / "weights.json").write_text(json.dumps({"block": BLOCK, "f32_bytes": f32_offset, "tensors": manifest}))
+    return {"graph_mb": round((out / "graph.onnx").stat().st_size / 1e6, 3), "weights_q8_mb": round(len(q8) / 1e6, 2),
+            "expanded_mb": round(f32_offset / 1e6, 2), "tensors": len(manifest)}
+
+
+def expand(out: Path) -> Path:
+    """What the browser does: rebuild weights.f32 from weights.q8. Used here to check accuracy."""
+    meta = json.loads((out / "weights.json").read_text())
+    raw = (out / "weights.q8").read_bytes()
+    f32 = np.zeros(meta["f32_bytes"] // 4, np.float32)
+    for t in meta["tensors"]:
+        q = np.frombuffer(raw, np.int8, t["count"], t["q_offset"])
+        blocks = (t["count"] + meta["block"] - 1) // meta["block"]
+        scales = np.frombuffer(raw, np.float32, blocks, t["scale_offset"])
+        f32[t["f32_offset"] // 4: t["f32_offset"] // 4 + t["count"]] = dequantize(q, scales, t["count"])
+    path = out / EXTERNAL
+    f32.tofile(path)
+    return path
+
+
+if __name__ == "__main__":
+    src, out = Path(sys.argv[1]), Path(sys.argv[2])
+    print(json.dumps(compress(src, out), indent=1))
+    expand(out)
+    print(f"expanded {out / EXTERNAL} for checking")

v2/eval.py

@@ -352,6 +352,30 @@ def hf_causal_lm(name: str, repo: str) -> TokenLM:
     return TokenLM(name, next_logits, encode, texts, stops)
 
 
+def onnx_token_lm(name: str, folder: str, fim: bool) -> TokenLM:
+    """A tinynext v2 export: graph.onnx + weights.f32 (compressed, expanded) or model.onnx, plus tokenizer.json."""
+    import onnxruntime as ort
+    from tokenizers import Tokenizer
+
+    folder = Path(folder)
+    graph = folder / "graph.onnx" if (folder / "graph.onnx").exists() else folder / "model.onnx"
+    session = ort.InferenceSession(str(graph), providers=["CPUExecutionProvider"])
+    config = json.loads((folder / "config.json").read_text())
+    tokenizer = Tokenizer.from_file(str(folder / "tokenizer.json"))
+    empty = np.zeros((config["layers"], 1, config["heads"], 0, config["dim"] // config["heads"]), np.float32)
+
+    def next_logits(ids, cache):
+        past_k, past_v = cache if cache else (empty, empty)
+        logits, k, v = session.run(None, {"input_ids": np.array([ids], np.int64), "past_k": past_k, "past_v": past_v})
+        return logits[0], (k, v)
+
+    special = len(config["special_tokens"])
+    texts = [tokenizer.decode([i], skip_special_tokens=False) if i >= special else "" for i in range(config["vocab"])]
+    encode = lambda text: tokenizer.encode(text).ids or [0]
+    prompt = (lambda prefix, suffix: prefix if not suffix else [2, *suffix, 1, *prefix, 3]) if fim else None
+    return TokenLM(name, next_logits, encode, texts, set(range(special)), prompt=prompt)
+
+
 # ---- Scoring ---------------------------------------------------------------------------------------------------
 
 
@@ -446,6 +470,8 @@ def main():
     parser.add_argument("--model", help="Name for the metrics file")
     parser.add_argument("--onnx", help="A character-level GRU exported like enron-autocomplete")
     parser.add_argument("--hf", help="A Hugging Face causal LM to evaluate, e.g. HuggingFaceTB/SmolLM2-135M")
+    parser.add_argument("--onnx-lm", help="Folder with a tinynext v2 ONNX export, config.json and tokenizer.json")
+    parser.add_argument("--no-fim", action="store_true", help="With --onnx-lm: the model wasn't trained to fill in the middle")
     parser.add_argument("--max-pauses", type=int, default=1000, help="Per domain and scenario")
     parser.add_argument("--table", action="store_true", help="Print a comparison of every metrics/*.json")
     args = parser.parse_args()
@@ -457,6 +483,8 @@ def main():
         model: Model = CharGRU(args.model, args.onnx)
     elif args.hf:
         model = hf_causal_lm(args.model, args.hf)
+    elif args.onnx_lm:
+        model = onnx_token_lm(args.model, args.onnx_lm, fim=not args.no_fim)
     else:
         parser.error("Pass --onnx, --hf or --table. train.py evaluates its own model through score().")
     all_pauses = pauses(load_eval_set(), args.max_pauses)

v2/export.py

@@ -0,0 +1,58 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = [
+#   "torch>=2.6,<2.8",
+#   "numpy<3",
+#   "tokenizers>=0.20",
+#   "onnx>=1.17",
+#   "onnxruntime>=1.20",
+#   "safetensors>=0.4",
+#   "huggingface_hub>=0.30",
+# ]
+# ///
+"""Re-exports a trained tinynext checkpoint to browser ONNX (float and int8) with train.py's export, without retraining.
+
+  uv run export.py --weights v2-release/v2 --out v2-release/browser
+"""
+
+import argparse
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import torch
+from safetensors.torch import load_file
+from tokenizers import Tokenizer
+
+HERE = Path(__file__).parent
+
+
+def load(name: str):
+    spec = importlib.util.spec_from_file_location(name, HERE / f"{'eval' if name == 'tinynext_eval' else name}.py")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--weights", required=True, help="Folder with model.safetensors, config.json, tokenizer.json")
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args()
+    train, ev = load("train"), load("tinynext_eval")
+    weights, out = Path(args.weights), Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    config = json.loads((weights / "config.json").read_text())
+    model = train.TinyNext(config["vocab"], config["dim"], config["layers"], config["heads"], config["ffn"])
+    model.load_state_dict(load_file(weights / "model.safetensors"))
+    tokenizer = Tokenizer.from_file(str(weights / "tokenizer.json"))
+    report = train.export(model, out, torch.device("cpu"), train.parity_prompts(ev, tokenizer))
+    (out / "export.json").write_text(json.dumps(report, indent=1))
+    print(json.dumps(report, indent=1))
+
+
+if __name__ == "__main__":
+    main()

v2/metrics/tinynext-v2-float-onnx.json

@@ -0,0 +1,411 @@
+{
+ "model": "tinynext-v2-float-onnx",
+ "sees_after": true,
+ "cutoffs": {
+  "0.3": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.1833,
+    "right": 0.5818,
+    "avg_right_chars": 6.03,
+    "multiword_shown": 0.0733,
+    "multiword_right": 0.4545,
+    "saved_with_tab": 0.0536,
+    "saved_word_by_word": 0.0594
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.13,
+    "right": 0.6923,
+    "avg_right_chars": 5.93,
+    "multiword_shown": 0.0533,
+    "multiword_right": 0.6875,
+    "saved_with_tab": 0.0444,
+    "saved_word_by_word": 0.0464
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.3067,
+    "right": 0.6087,
+    "avg_right_chars": 7.71,
+    "multiword_shown": 0.1933,
+    "multiword_right": 0.569,
+    "saved_with_tab": 0.12,
+    "saved_word_by_word": 0.1408
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.2833,
+    "right": 0.5059,
+    "avg_right_chars": 6.53,
+    "multiword_shown": 0.14,
+    "multiword_right": 0.5,
+    "saved_with_tab": 0.0781,
+    "saved_word_by_word": 0.0919
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.2367,
+    "right": 0.5915,
+    "avg_right_chars": 5.43,
+    "multiword_shown": 0.0667,
+    "multiword_right": 0.45,
+    "saved_with_tab": 0.0633,
+    "saved_word_by_word": 0.0722
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.2367,
+    "right": 0.5915,
+    "avg_right_chars": 5.57,
+    "multiword_shown": 0.0933,
+    "multiword_right": 0.5357,
+    "saved_with_tab": 0.065,
+    "saved_word_by_word": 0.0703
+   }
+  },
+  "0.4": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.15,
+    "right": 0.6889,
+    "avg_right_chars": 6.03,
+    "multiword_shown": 0.0567,
+    "multiword_right": 0.5882,
+    "saved_with_tab": 0.0519,
+    "saved_word_by_word": 0.055
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.0967,
+    "right": 0.7586,
+    "avg_right_chars": 5.82,
+    "multiword_shown": 0.0367,
+    "multiword_right": 0.9091,
+    "saved_with_tab": 0.0356,
+    "saved_word_by_word": 0.0358
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.2367,
+    "right": 0.7465,
+    "avg_right_chars": 7.23,
+    "multiword_shown": 0.13,
+    "multiword_right": 0.6667,
+    "saved_with_tab": 0.1064,
+    "saved_word_by_word": 0.1161
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.17,
+    "right": 0.6275,
+    "avg_right_chars": 6.38,
+    "multiword_shown": 0.0733,
+    "multiword_right": 0.6364,
+    "saved_with_tab": 0.0567,
+    "saved_word_by_word": 0.0658
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.19,
+    "right": 0.7368,
+    "avg_right_chars": 5.48,
+    "multiword_shown": 0.0467,
+    "multiword_right": 0.5714,
+    "saved_with_tab": 0.0639,
+    "saved_word_by_word": 0.0678
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.16,
+    "right": 0.6875,
+    "avg_right_chars": 5.27,
+    "multiword_shown": 0.06,
+    "multiword_right": 0.6111,
+    "saved_with_tab": 0.0483,
+    "saved_word_by_word": 0.0514
+   }
+  },
+  "0.5": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.1133,
+    "right": 0.7647,
+    "avg_right_chars": 5.92,
+    "multiword_shown": 0.0367,
+    "multiword_right": 0.6364,
+    "saved_with_tab": 0.0428,
+    "saved_word_by_word": 0.0436
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.0833,
+    "right": 0.8,
+    "avg_right_chars": 5.4,
+    "multiword_shown": 0.0267,
+    "multiword_right": 0.875,
+    "saved_with_tab": 0.03,
+    "saved_word_by_word": 0.0303
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.1767,
+    "right": 0.8113,
+    "avg_right_chars": 7.12,
+    "multiword_shown": 0.0967,
+    "multiword_right": 0.7241,
+    "saved_with_tab": 0.085,
+    "saved_word_by_word": 0.0917
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.1233,
+    "right": 0.7297,
+    "avg_right_chars": 6.37,
+    "multiword_shown": 0.0567,
+    "multiword_right": 0.6471,
+    "saved_with_tab": 0.0478,
+    "saved_word_by_word": 0.0561
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.1367,
+    "right": 0.8049,
+    "avg_right_chars": 5.48,
+    "multiword_shown": 0.0267,
+    "multiword_right": 0.75,
+    "saved_with_tab": 0.0503,
+    "saved_word_by_word": 0.0514
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.13,
+    "right": 0.7436,
+    "avg_right_chars": 5.07,
+    "multiword_shown": 0.0367,
+    "multiword_right": 0.5455,
+    "saved_with_tab": 0.0408,
+    "saved_word_by_word": 0.0431
+   }
+  },
+  "0.6": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.1,
+    "right": 0.8,
+    "avg_right_chars": 6.0,
+    "multiword_shown": 0.0267,
+    "multiword_right": 0.75,
+    "saved_with_tab": 0.04,
+    "saved_word_by_word": 0.04
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.0533,
+    "right": 0.9375,
+    "avg_right_chars": 5.67,
+    "multiword_shown": 0.02,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0236,
+    "saved_word_by_word": 0.0236
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.14,
+    "right": 0.881,
+    "avg_right_chars": 6.57,
+    "multiword_shown": 0.0733,
+    "multiword_right": 0.8182,
+    "saved_with_tab": 0.0675,
+    "saved_word_by_word": 0.0708
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.0767,
+    "right": 0.7391,
+    "avg_right_chars": 6.18,
+    "multiword_shown": 0.04,
+    "multiword_right": 0.5,
+    "saved_with_tab": 0.0292,
+    "saved_word_by_word": 0.0375
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.0967,
+    "right": 0.8966,
+    "avg_right_chars": 5.73,
+    "multiword_shown": 0.02,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0414,
+    "saved_word_by_word": 0.0414
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.0833,
+    "right": 0.92,
+    "avg_right_chars": 4.96,
+    "multiword_shown": 0.0133,
+    "multiword_right": 0.75,
+    "saved_with_tab": 0.0317,
+    "saved_word_by_word": 0.0328
+   }
+  },
+  "0.7": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.07,
+    "right": 0.9524,
+    "avg_right_chars": 5.7,
+    "multiword_shown": 0.0167,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0317,
+    "saved_word_by_word": 0.0317
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.03,
+    "right": 0.8889,
+    "avg_right_chars": 5.12,
+    "multiword_shown": 0.0033,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0114,
+    "saved_word_by_word": 0.0114
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.11,
+    "right": 0.9394,
+    "avg_right_chars": 6.23,
+    "multiword_shown": 0.05,
+    "multiword_right": 0.8667,
+    "saved_with_tab": 0.0536,
+    "saved_word_by_word": 0.0544
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.0633,
+    "right": 0.7895,
+    "avg_right_chars": 6.13,
+    "multiword_shown": 0.0333,
+    "multiword_right": 0.6,
+    "saved_with_tab": 0.0256,
+    "saved_word_by_word": 0.0328
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.0667,
+    "right": 0.85,
+    "avg_right_chars": 5.41,
+    "multiword_shown": 0.0067,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0256,
+    "saved_word_by_word": 0.0256
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.06,
+    "right": 0.9444,
+    "avg_right_chars": 4.71,
+    "multiword_shown": 0.0033,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0222,
+    "saved_word_by_word": 0.0222
+   }
+  }
+ },
+ "examples": [
+  {
+   "domain": "articles",
+   "scenario": "typing",
+   "before": "elipads, IFR approaches, and a separate float plane dock alo",
+   "suggestion": "ng the",
+   "truth": "ng the Fraser Rive",
+   "right": true
+  },
+  {
+   "domain": "articles",
+   "scenario": "editing",
+   "before": " where he met his wife, Kisa, a graduate of the University o",
+   "suggestion": "f Illinois",
+   "truth": "f Colorado. Sasaki the",
+   "right": false
+  },
+  {
+   "domain": "articles",
+   "scenario": "editing",
+   "before": "ing any kind of direct discrimination that was prominent els",
+   "suggestion": "ewhere.",
+   "truth": "ewhere in the count",
+   "right": false
+  },
+  {
+   "domain": "messages",
+   "scenario": "typing",
+   "before": "T's ability to generate coherent and contextually relevant r",
+   "suggestion": "esponses",
+   "truth": "esponses is based on",
+   "right": true
+  },
+  {
+   "domain": "messages",
+   "scenario": "typing",
+   "before": " prioritize the development of tools and models that are des",
+   "suggestion": "igned to",
+   "truth": "igned to be inclusiv",
+   "right": true
+  },
+  {
+   "domain": "messages",
+   "scenario": "editing",
+   "before": "e at which a population is growing.\nL'Hopital's rule can als",
+   "suggestion": "o be used to",
+   "truth": "o be used to solve probl",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "typing",
+   "before": "lt time but am confident that I would make an excellent addi",
+   "suggestion": "tion to",
+   "truth": "tion to your team f",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "typing",
+   "before": ". I recommend we agree. Although, we are in the content deve",
+   "suggestion": "lopment",
+   "truth": "lopment phase the t",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "editing",
+   "before": "ured products and marketing of electricity for the United St",
+   "suggestion": "ates",
+   "truth": "ates ? Actively ",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "editing",
+   "before": "I'm not sure what you are looking for. The Early Termination",
+   "suggestion": " Log",
+   "truth": " Date is one of ",
+   "right": false
+  },
+  {
+   "domain": "email",
+   "scenario": "editing",
+   "before": " action is necessary as contractually we are coming to the e",
+   "suggestion": "nd of",
+   "truth": "nd of the accepta",
+   "right": true
+  }
+ ],
+ "latency_ms": {
+  "p50": 25.37,
+  "p95": 77.65,
+  "note": "Measured in this eval process, not in a browser."
+ }
+}

v2/metrics/tinynext-v2-q8-onnx.json

@@ -0,0 +1,411 @@
+{
+ "model": "tinynext-v2-q8-onnx",
+ "sees_after": true,
+ "cutoffs": {
+  "0.3": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.1767,
+    "right": 0.5849,
+    "avg_right_chars": 6.0,
+    "multiword_shown": 0.0733,
+    "multiword_right": 0.4545,
+    "saved_with_tab": 0.0517,
+    "saved_word_by_word": 0.0575
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.13,
+    "right": 0.6923,
+    "avg_right_chars": 5.93,
+    "multiword_shown": 0.0533,
+    "multiword_right": 0.6875,
+    "saved_with_tab": 0.0444,
+    "saved_word_by_word": 0.0464
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.3,
+    "right": 0.6333,
+    "avg_right_chars": 7.65,
+    "multiword_shown": 0.1933,
+    "multiword_right": 0.569,
+    "saved_with_tab": 0.1211,
+    "saved_word_by_word": 0.1403
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.2767,
+    "right": 0.5181,
+    "avg_right_chars": 6.53,
+    "multiword_shown": 0.14,
+    "multiword_right": 0.5,
+    "saved_with_tab": 0.0781,
+    "saved_word_by_word": 0.0919
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.2333,
+    "right": 0.5857,
+    "avg_right_chars": 5.46,
+    "multiword_shown": 0.0667,
+    "multiword_right": 0.45,
+    "saved_with_tab": 0.0622,
+    "saved_word_by_word": 0.0714
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.2333,
+    "right": 0.6,
+    "avg_right_chars": 5.57,
+    "multiword_shown": 0.0933,
+    "multiword_right": 0.5357,
+    "saved_with_tab": 0.065,
+    "saved_word_by_word": 0.0703
+   }
+  },
+  "0.4": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.1467,
+    "right": 0.6818,
+    "avg_right_chars": 6.0,
+    "multiword_shown": 0.0567,
+    "multiword_right": 0.5882,
+    "saved_with_tab": 0.05,
+    "saved_word_by_word": 0.0531
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.0967,
+    "right": 0.7586,
+    "avg_right_chars": 5.82,
+    "multiword_shown": 0.0367,
+    "multiword_right": 0.9091,
+    "saved_with_tab": 0.0356,
+    "saved_word_by_word": 0.0358
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.24,
+    "right": 0.7361,
+    "avg_right_chars": 7.26,
+    "multiword_shown": 0.1333,
+    "multiword_right": 0.65,
+    "saved_with_tab": 0.1069,
+    "saved_word_by_word": 0.1167
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.17,
+    "right": 0.6275,
+    "avg_right_chars": 6.38,
+    "multiword_shown": 0.0733,
+    "multiword_right": 0.6364,
+    "saved_with_tab": 0.0567,
+    "saved_word_by_word": 0.0658
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.1867,
+    "right": 0.75,
+    "avg_right_chars": 5.48,
+    "multiword_shown": 0.0467,
+    "multiword_right": 0.5714,
+    "saved_with_tab": 0.0639,
+    "saved_word_by_word": 0.0678
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.1667,
+    "right": 0.68,
+    "avg_right_chars": 5.26,
+    "multiword_shown": 0.06,
+    "multiword_right": 0.6111,
+    "saved_with_tab": 0.0497,
+    "saved_word_by_word": 0.0528
+   }
+  },
+  "0.5": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.11,
+    "right": 0.7879,
+    "avg_right_chars": 5.81,
+    "multiword_shown": 0.0333,
+    "multiword_right": 0.7,
+    "saved_with_tab": 0.0419,
+    "saved_word_by_word": 0.0425
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.0833,
+    "right": 0.8,
+    "avg_right_chars": 5.4,
+    "multiword_shown": 0.0267,
+    "multiword_right": 0.875,
+    "saved_with_tab": 0.03,
+    "saved_word_by_word": 0.0303
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.18,
+    "right": 0.7963,
+    "avg_right_chars": 6.98,
+    "multiword_shown": 0.1,
+    "multiword_right": 0.7,
+    "saved_with_tab": 0.0833,
+    "saved_word_by_word": 0.09
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.1233,
+    "right": 0.7297,
+    "avg_right_chars": 6.37,
+    "multiword_shown": 0.0567,
+    "multiword_right": 0.6471,
+    "saved_with_tab": 0.0478,
+    "saved_word_by_word": 0.0561
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.14,
+    "right": 0.7857,
+    "avg_right_chars": 5.48,
+    "multiword_shown": 0.0267,
+    "multiword_right": 0.75,
+    "saved_with_tab": 0.0503,
+    "saved_word_by_word": 0.0514
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.1267,
+    "right": 0.7632,
+    "avg_right_chars": 5.07,
+    "multiword_shown": 0.0367,
+    "multiword_right": 0.5455,
+    "saved_with_tab": 0.0408,
+    "saved_word_by_word": 0.0431
+   }
+  },
+  "0.6": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.0967,
+    "right": 0.7931,
+    "avg_right_chars": 5.96,
+    "multiword_shown": 0.0267,
+    "multiword_right": 0.75,
+    "saved_with_tab": 0.0381,
+    "saved_word_by_word": 0.0381
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.05,
+    "right": 0.9333,
+    "avg_right_chars": 5.79,
+    "multiword_shown": 0.02,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0225,
+    "saved_word_by_word": 0.0225
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.1467,
+    "right": 0.8864,
+    "avg_right_chars": 6.54,
+    "multiword_shown": 0.0733,
+    "multiword_right": 0.8182,
+    "saved_with_tab": 0.0708,
+    "saved_word_by_word": 0.0742
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.0767,
+    "right": 0.7391,
+    "avg_right_chars": 6.18,
+    "multiword_shown": 0.04,
+    "multiword_right": 0.5,
+    "saved_with_tab": 0.0292,
+    "saved_word_by_word": 0.0375
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.1,
+    "right": 0.8667,
+    "avg_right_chars": 5.73,
+    "multiword_shown": 0.02,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0414,
+    "saved_word_by_word": 0.0414
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.09,
+    "right": 0.8889,
+    "avg_right_chars": 4.92,
+    "multiword_shown": 0.0133,
+    "multiword_right": 0.75,
+    "saved_with_tab": 0.0328,
+    "saved_word_by_word": 0.0339
+   }
+  },
+  "0.7": {
+   "articles/typing": {
+    "pauses": 300,
+    "shown": 0.0667,
+    "right": 0.95,
+    "avg_right_chars": 5.63,
+    "multiword_shown": 0.0167,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0297,
+    "saved_word_by_word": 0.0297
+   },
+   "articles/editing": {
+    "pauses": 300,
+    "shown": 0.03,
+    "right": 0.8889,
+    "avg_right_chars": 5.12,
+    "multiword_shown": 0.0033,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0114,
+    "saved_word_by_word": 0.0114
+   },
+   "messages/typing": {
+    "pauses": 300,
+    "shown": 0.1133,
+    "right": 0.9412,
+    "avg_right_chars": 6.22,
+    "multiword_shown": 0.05,
+    "multiword_right": 0.8667,
+    "saved_with_tab": 0.0553,
+    "saved_word_by_word": 0.0561
+   },
+   "messages/editing": {
+    "pauses": 300,
+    "shown": 0.0633,
+    "right": 0.7895,
+    "avg_right_chars": 5.93,
+    "multiword_shown": 0.03,
+    "multiword_right": 0.5556,
+    "saved_with_tab": 0.0247,
+    "saved_word_by_word": 0.0319
+   },
+   "email/typing": {
+    "pauses": 300,
+    "shown": 0.0667,
+    "right": 0.85,
+    "avg_right_chars": 5.41,
+    "multiword_shown": 0.0067,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0256,
+    "saved_word_by_word": 0.0256
+   },
+   "email/editing": {
+    "pauses": 300,
+    "shown": 0.06,
+    "right": 0.9444,
+    "avg_right_chars": 4.71,
+    "multiword_shown": 0.0033,
+    "multiword_right": 1.0,
+    "saved_with_tab": 0.0222,
+    "saved_word_by_word": 0.0222
+   }
+  }
+ },
+ "examples": [
+  {
+   "domain": "articles",
+   "scenario": "typing",
+   "before": "elipads, IFR approaches, and a separate float plane dock alo",
+   "suggestion": "ng the",
+   "truth": "ng the Fraser Rive",
+   "right": true
+  },
+  {
+   "domain": "articles",
+   "scenario": "editing",
+   "before": " where he met his wife, Kisa, a graduate of the University o",
+   "suggestion": "f Illinois",
+   "truth": "f Colorado. Sasaki the",
+   "right": false
+  },
+  {
+   "domain": "articles",
+   "scenario": "editing",
+   "before": "ing any kind of direct discrimination that was prominent els",
+   "suggestion": "ewhere.",
+   "truth": "ewhere in the count",
+   "right": false
+  },
+  {
+   "domain": "messages",
+   "scenario": "typing",
+   "before": "T's ability to generate coherent and contextually relevant r",
+   "suggestion": "esponses",
+   "truth": "esponses is based on",
+   "right": true
+  },
+  {
+   "domain": "messages",
+   "scenario": "typing",
+   "before": " prioritize the development of tools and models that are des",
+   "suggestion": "igned to",
+   "truth": "igned to be inclusiv",
+   "right": true
+  },
+  {
+   "domain": "messages",
+   "scenario": "editing",
+   "before": "e at which a population is growing.\nL'Hopital's rule can als",
+   "suggestion": "o be used to",
+   "truth": "o be used to solve probl",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "typing",
+   "before": "lt time but am confident that I would make an excellent addi",
+   "suggestion": "tion to",
+   "truth": "tion to your team f",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "typing",
+   "before": ". I recommend we agree. Although, we are in the content deve",
+   "suggestion": "lopment",
+   "truth": "lopment phase the t",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "editing",
+   "before": "ured products and marketing of electricity for the United St",
+   "suggestion": "ates",
+   "truth": "ates ? Actively ",
+   "right": true
+  },
+  {
+   "domain": "email",
+   "scenario": "editing",
+   "before": "I'm not sure what you are looking for. The Early Termination",
+   "suggestion": " Log",
+   "truth": " Date is one of ",
+   "right": false
+  },
+  {
+   "domain": "email",
+   "scenario": "editing",
+   "before": " action is necessary as contractually we are coming to the e",
+   "suggestion": "nd of",
+   "truth": "nd of the accepta",
+   "right": true
+  }
+ ],
+ "latency_ms": {
+  "p50": 25.25,
+  "p95": 75.94,
+  "note": "Measured in this eval process, not in a browser."
+ }
+}

v2/train.py

@@ -275,6 +275,7 @@ class TinyNext(nn.Module):
         self.embed = nn.Embedding(vocab, dim)
         self.blocks = nn.ModuleList(Block(dim, heads, ffn) for _ in range(layers))
         self.norm = RMSNorm(dim)
+        self.head = None  # set only for export: an untied copy of the output weights, so int8 quantization covers it
         self.head_dim = dim // heads
         inverse = 1.0 / (10000 ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim))
         self.register_buffer("inverse_frequency", inverse, persistent=False)
@@ -298,7 +299,7 @@ class TinyNext(nn.Module):
             x, k, v = block(x, cos, sin, None if past_k is None else past_k[index], None if past_v is None else past_v[index], mask)
             new_k.append(k)
             new_v.append(v)
-        logits = self.norm(x) @ self.embed.weight.T
+        logits = self.norm(x) @ (self.embed.weight.T if self.head is None else self.head)
         return logits, torch.stack(new_k), torch.stack(new_v)
 
 
@@ -414,46 +415,79 @@ def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device, fim: bool = T
     return ev.TokenLM(name, next_logits, encode, texts, set(range(len(SPECIAL))), prompt=prompt)
 
 
-def export(model: TinyNext, output: Path, device) -> dict:
+def parity_prompts(ev, tokenizer, count: int = 300) -> list[list[int]]:
+    """Real text from the eval set, cut at random points, as token prompts."""
+    rng = random.Random(0)
+    texts = [p for domain in ev.load_eval_set().values() for doc in domain for p in doc]
+    prompts = []
+    for text in rng.sample(texts, min(count, len(texts))):
+        ids = tokenizer.encode(text[: rng.randint(40, len(text))]).ids[-300:]
+        if len(ids) >= 4:
+            prompts.append(ids)
+    return prompts
+
+
+def greedy_tokens(step, ids: list[int], empty, steps: int) -> list[int]:
+    """Prompt, then `steps` cached greedy steps; returns the top token at each point."""
+    logits, k, v = step(np.array([ids]), empty, empty)
+    tops = [int(logits.argmax())]
+    for _ in range(steps):
+        logits, k, v = step(np.array([[tops[-1]]]), k, v)
+        tops.append(int(logits.argmax()))
+    return tops
+
+
+def export(model: TinyNext, output: Path, device, prompts: list[list[int]]) -> dict:
+    """Float and int8 ONNX for the browser, checked against PyTorch on real prompts over several cached steps."""
     import onnxruntime as ort
     from onnxruntime.quantization import QuantType, quantize_dynamic
 
     model = model.float().eval().to("cpu")
+    model.head = nn.Parameter(model.embed.weight.detach().T.contiguous().clone())
     cfg = model.config
     head_dim = cfg["dim"] // cfg["heads"]
-    ids = torch.tensor([[5, 6, 7]], dtype=torch.long)
-    empty = torch.zeros(cfg["layers"], 1, cfg["heads"], 0, head_dim)
-    # Trace with a non-empty cache so no shape gets fixed at zero; the checks below run with an empty one.
+    empty = np.zeros((cfg["layers"], 1, cfg["heads"], 0, head_dim), np.float32)
+    # Trace with a non-empty cache so no shape gets fixed at zero; the checks below start from an empty one.
     with torch.no_grad():
         _, trace_k, trace_v = model(torch.tensor([[1, 2]]))
     float_path, int8_path = output / "model.onnx", output / "model_int8.onnx"
     torch.onnx.export(
-        BrowserStep(model), (ids, trace_k, trace_v), float_path, opset_version=17,
+        BrowserStep(model), (torch.tensor([[5, 6, 7]]), trace_k, trace_v), float_path, opset_version=17,
         input_names=["input_ids", "past_k", "past_v"], output_names=["logits", "present_k", "present_v"],
         dynamic_axes={"input_ids": {1: "length"}, "past_k": {3: "past"}, "past_v": {3: "past"},
                       "present_k": {3: "total"}, "present_v": {3: "total"}},
         dynamo=False,
     )
-    quantize_dynamic(float_path, int8_path, weight_type=QuantType.QInt8, op_types_to_quantize=["MatMul"])
+    quantize_dynamic(float_path, int8_path, weight_type=QuantType.QInt8, op_types_to_quantize=["MatMul", "Gather"], per_channel=True)
 
-    report = {}
-    with torch.no_grad():
-        expected, k, v = model(ids)
-        expected_next, _, _ = model(torch.tensor([[8]]), k, v)
+    @torch.no_grad()
+    def torch_step(ids, k, v):
+        past = None if k.shape[3] == 0 else (torch.from_numpy(k), torch.from_numpy(v))
+        logits, nk, nv = model(torch.from_numpy(ids), *(past or (None, None)))
+        return logits[0, -1].numpy(), nk.numpy(), nv.numpy()
+
+    expected = [greedy_tokens(torch_step, ids, empty, 4) for ids in prompts]
+    report = {"parity_prompts": len(prompts), "parity_steps_per_prompt": 5}
     for name, path in (("float", float_path), ("int8", int8_path)):
         session = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"])
-        logits, pk, pv = session.run(None, {"input_ids": ids.numpy(), "past_k": empty.numpy(), "past_v": empty.numpy()})
-        logits2, _, _ = session.run(None, {"input_ids": np.array([[8]]), "past_k": pk, "past_v": pv})
+        onnx_step = lambda ids, k, v: (lambda out: (out[0][0], out[1], out[2]))(
+            session.run(None, {"input_ids": ids.astype(np.int64), "past_k": k, "past_v": v}))
+        got = [greedy_tokens(onnx_step, ids, empty, 4) for ids in prompts]
+        first = np.mean([g[0] == e[0] for g, e in zip(got, expected)])
+        all_steps = np.mean([g == e for g, e in zip(got, expected)])
+        _, pk, pv = onnx_step(np.array([prompts[0]]), empty, empty)
         timings = []
         for _ in range(200):
             began = time.perf_counter()
-            session.run(None, {"input_ids": np.array([[8]]), "past_k": pk, "past_v": pv})
+            onnx_step(np.array([[8]]), pk, pv)
             timings.append((time.perf_counter() - began) * 1000)
         report[name] = {
             "megabytes": round(path.stat().st_size / 1e6, 2),
-            "same_top_token": bool(logits.argmax() == expected[0, -1].argmax() and logits2.argmax() == expected_next[0, -1].argmax()),
+            "same_first_token": round(float(first), 4),
+            "same_five_greedy_tokens": round(float(all_steps), 4),
             "cached_step_ms_p50_cpu": round(float(np.median(timings)), 3),
         }
+    model.head = None
     model.to(device)
     return report
 
@@ -620,7 +654,7 @@ def main():
     (output / "metrics").mkdir(exist_ok=True)
     (output / "metrics" / f"{args.name}.json").write_text(json.dumps(scored, indent=1))
     tracker.log(headline(scored))
-    exported = export(model, output, device)
+    exported = export(model, output, device, parity_prompts(ev, tokenizer))
 
     config = {
         **model.config, "parameters": parameters, "seq": args.seq, "special_tokens": dict(zip(SPECIAL, range(len(SPECIAL)))),

v2/web/config.json

@@ -0,0 +1,16 @@
+{
+ "vocab": 8192,
+ "dim": 320,
+ "layers": 8,
+ "heads": 5,
+ "ffn": 864,
+ "parameters": 12539200,
+ "seq": 512,
+ "special_tokens": {
+  "<|endoftext|>": 0,
+  "<|pre|>": 1,
+  "<|suf|>": 2,
+  "<|mid|>": 3
+ },
+ "prompt": "No text after the cursor: the text before it. Otherwise: <|suf|> after <|pre|> before <|mid|>."
+}

v2/web/graph.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e63c34d5f6882b6c9d7dda768ea6009edc831f26f5e7a800161a54a204f521d7
+size 194807

v2/web/parity.json

@@ -0,0 +1,16 @@
+{
+ "parity_prompts": 300,
+ "parity_steps_per_prompt": 5,
+ "float": {
+  "megabytes": 60.81,
+  "same_first_token": 1.0,
+  "same_five_greedy_tokens": 1.0,
+  "cached_step_ms_p50_cpu": 1.2
+ },
+ "int8": {
+  "megabytes": 15.64,
+  "same_first_token": 0.55,
+  "same_five_greedy_tokens": 0.33,
+  "cached_step_ms_p50_cpu": 0.71
+ }
+}

v2/web/weights.json

@@ -0,0 +1 @@
+{"block": 64, "f32_bytes": 50135040, "tensors": [{"name": "model.embed.weight", "shape": [8192, 320], "count": 2621440, "f32_offset": 0, "q_offset": 0, "scale_offset": 2621440}, {"name": "onnx::MatMul_1554", "shape": [320, 960], "count": 307200, "f32_offset": 10485760, "q_offset": 2785280, "scale_offset": 3092480}, {"name": "onnx::MatMul_1571", "shape": [320, 320], "count": 102400, "f32_offset": 11714560, "q_offset": 3111680, "scale_offset": 3214080}, {"name": "onnx::MatMul_1572", "shape": [320, 1728], "count": 552960, "f32_offset": 12124160, "q_offset": 3220480, "scale_offset": 3773440}, {"name": "onnx::MatMul_1573", "shape": [864, 320], "count": 276480, "f32_offset": 14336000, "q_offset": 3808000, "scale_offset": 4084480}, {"name": "onnx::MatMul_1574", "shape": [320, 960], "count": 307200, "f32_offset": 15441920, "q_offset": 4101760, "scale_offset": 4408960}, {"name": "onnx::MatMul_1591", "shape": [320, 320], "count": 102400, "f32_offset": 16670720, "q_offset": 4428160, "scale_offset": 4530560}, {"name": "onnx::MatMul_1592", "shape": [320, 1728], "count": 552960, "f32_offset": 17080320, "q_offset": 4536960, "scale_offset": 5089920}, {"name": "onnx::MatMul_1593", "shape": [864, 320], "count": 276480, "f32_offset": 19292160, "q_offset": 5124480, "scale_offset": 5400960}, {"name": "onnx::MatMul_1594", "shape": [320, 960], "count": 307200, "f32_offset": 20398080, "q_offset": 5418240, "scale_offset": 5725440}, {"name": "onnx::MatMul_1611", "shape": [320, 320], "count": 102400, "f32_offset": 21626880, "q_offset": 5744640, "scale_offset": 5847040}, {"name": "onnx::MatMul_1612", "shape": [320, 1728], "count": 552960, "f32_offset": 22036480, "q_offset": 5853440, "scale_offset": 6406400}, {"name": "onnx::MatMul_1613", "shape": [864, 320], "count": 276480, "f32_offset": 24248320, "q_offset": 6440960, "scale_offset": 6717440}, {"name": "onnx::MatMul_1614", "shape": [320, 960], "count": 307200, "f32_offset": 25354240, "q_offset": 6734720, "scale_offset": 7041920}, {"name": "onnx::MatMul_1631", "shape": [320, 320], "count": 102400, "f32_offset": 26583040, "q_offset": 7061120, "scale_offset": 7163520}, {"name": "onnx::MatMul_1632", "shape": [320, 1728], "count": 552960, "f32_offset": 26992640, "q_offset": 7169920, "scale_offset": 7722880}, {"name": "onnx::MatMul_1633", "shape": [864, 320], "count": 276480, "f32_offset": 29204480, "q_offset": 7757440, "scale_offset": 8033920}, {"name": "onnx::MatMul_1634", "shape": [320, 960], "count": 307200, "f32_offset": 30310400, "q_offset": 8051200, "scale_offset": 8358400}, {"name": "onnx::MatMul_1651", "shape": [320, 320], "count": 102400, "f32_offset": 31539200, "q_offset": 8377600, "scale_offset": 8480000}, {"name": "onnx::MatMul_1652", "shape": [320, 1728], "count": 552960, "f32_offset": 31948800, "q_offset": 8486400, "scale_offset": 9039360}, {"name": "onnx::MatMul_1653", "shape": [864, 320], "count": 276480, "f32_offset": 34160640, "q_offset": 9073920, "scale_offset": 9350400}, {"name": "onnx::MatMul_1654", "shape": [320, 960], "count": 307200, "f32_offset": 35266560, "q_offset": 9367680, "scale_offset": 9674880}, {"name": "onnx::MatMul_1671", "shape": [320, 320], "count": 102400, "f32_offset": 36495360, "q_offset": 9694080, "scale_offset": 9796480}, {"name": "onnx::MatMul_1672", "shape": [320, 1728], "count": 552960, "f32_offset": 36904960, "q_offset": 9802880, "scale_offset": 10355840}, {"name": "onnx::MatMul_1673", "shape": [864, 320], "count": 276480, "f32_offset": 39116800, "q_offset": 10390400, "scale_offset": 10666880}, {"name": "onnx::MatMul_1674", "shape": [320, 960], "count": 307200, "f32_offset": 40222720, "q_offset": 10684160, "scale_offset": 10991360}, {"name": "onnx::MatMul_1691", "shape": [320, 320], "count": 102400, "f32_offset": 41451520, "q_offset": 11010560, "scale_offset": 11112960}, {"name": "onnx::MatMul_1692", "shape": [320, 1728], "count": 552960, "f32_offset": 41861120, "q_offset": 11119360, "scale_offset": 11672320}, {"name": "onnx::MatMul_1693", "shape": [864, 320], "count": 276480, "f32_offset": 44072960, "q_offset": 11706880, "scale_offset": 11983360}, {"name": "onnx::MatMul_1694", "shape": [320, 960], "count": 307200, "f32_offset": 45178880, "q_offset": 12000640, "scale_offset": 12307840}, {"name": "onnx::MatMul_1711", "shape": [320, 320], "count": 102400, "f32_offset": 46407680, "q_offset": 12327040, "scale_offset": 12429440}, {"name": "onnx::MatMul_1712", "shape": [320, 1728], "count": 552960, "f32_offset": 46817280, "q_offset": 12435840, "scale_offset": 12988800}, {"name": "onnx::MatMul_1713", "shape": [864, 320], "count": 276480, "f32_offset": 49029120, "q_offset": 13023360, "scale_offset": 13299840}]}

v2/web/weights.q8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b74d19232900fda1c0e1cdcd1113a4825cc05874805796e138edf778b8053dde
+size 13317120