"""Measure next-text bits per UTF-8 byte on exactly the same strings with either vocabulary.""" import argparse import hashlib import json import math from pathlib import Path import re import numpy as np import torch from torch.nn import functional as F from tokenizers import Tokenizer, decoders from safetensors.torch import load_file from franok.model import Config, Franok from tools.reading import PEERS def sample(data, tokenizer): rng = np.random.default_rng(4419073) for lang in ("en", "tr"): for name, pattern in ( ("web", r"val-\d+\.bin"), ("web-more", r"val-web-more-\d+\.bin"), ("stories", r"val-stories-\d+\.bin"), ): files = sorted( f for f in (data / lang).glob("*.bin") if re.fullmatch(pattern, f.name) and f.stat().st_size > 1100 ) if len(files) < 2: raise ValueError("Need several held-out shards for " + name) # The first shard is used by the training-time checkpoint selector. files = files[1:] seen = set() attempts = 0 while len(seen) < 32: attempts += 1 if attempts > 10000: raise ValueError("Insufficient intact source passages") file = files[rng.integers(len(files))] ids = np.memmap(file, dtype="= 7] text = tokenizer.decode(segment.tolist()) words = list(re.finditer(r"\S+", text)) if len(words) < 60: continue lo, edge, end = words[1].start(), words[18].end(), words[58].end() prefix, suffix = text[lo:edge], text[edge:end] digest = hashlib.sha256((prefix + suffix).encode()).hexdigest() if ( digest in seen or "�" in prefix + suffix or len(suffix.encode()) > 900 ): continue seen.add(digest) yield ( lang, name, prefix, suffix, dict(file=file.name, offset=at, sha256=digest), ) @torch.inference_mode() def main(): p = argparse.ArgumentParser() g = p.add_mutually_exclusive_group(required=True) g.add_argument("--run", type=Path) g.add_argument("--peer", nargs="?", const="smollm2", choices=PEERS) p.add_argument("--data", type=Path, required=True) p.add_argument("--out", type=Path, required=True) p.add_argument("--threads", type=int, default=3) a = p.parse_args() torch.set_num_threads(a.threads) franok_tokenizer = Tokenizer.from_file("web/tokenizer.json") franok_tokenizer.decoder = decoders.ByteLevel() prompt_prefix = [] if a.peer: from transformers import AutoTokenizer, AutoModelForCausalLM repo, revision = PEERS[a.peer] tk = AutoTokenizer.from_pretrained( repo, revision=revision, local_files_only=True ) model = AutoModelForCausalLM.from_pretrained( repo, revision=revision, local_files_only=True, dtype=torch.float32 ).eval() ident = revision if tk.encode("", add_special_tokens=True) == [tk.bos_token_id]: prompt_prefix = [tk.bos_token_id] def encode(s): return tk.encode(s, add_special_tokens=False) def decode(v): return tk.decode(v, clean_up_tokenization_spaces=False) def predict(x): return model(x, use_cache=False).logits else: model = Franok(Config(**json.loads((a.run / "config.json").read_text()))).eval() file = a.run / "model.safetensors" with file.open("rb") as f: ident = hashlib.file_digest(f, "sha256").hexdigest() model.load_state_dict(load_file(str(file))) def encode(s): return franok_tokenizer.encode(s, add_special_tokens=False).ids def decode(v): return franok_tokenizer.decode(v) predict = model result = {} rows = [] for lang, name, pre, suffix, source in sample(a.data, franok_tokenizer): head = encode(pre) allids = encode(pre + suffix) split_at_boundary = allids[: len(head)] != head if split_at_boundary: # Some vocabularies merge punctuation and a following newline. The # prefix is already visible at prediction time, so keep it intact. allids = head + encode(suffix) if not head or decode(head) != pre or decode(allids) != pre + suffix: raise RuntimeError( "The two text spans must tokenize and decode exactly: " + source["sha256"] ) context = prompt_prefix + allids[:-1] scores = predict(torch.tensor([context]))[0, len(prompt_prefix) + len(head) - 1 :].float() total = F.cross_entropy( scores, torch.tensor(allids[len(head) :]), reduction="sum" ).item() / math.log(2) size = len(suffix.encode()) rows.append( dict( lang=lang, collection=name, **source, bytes=size, bits=total, tokens=len(allids) - len(head), split_at_boundary=split_at_boundary, ) ) for key in (lang, lang + "-" + name): sums = result.setdefault(key, dict(windows=0, bytes=0, bits=0)) sums["windows"] += 1 sums["bytes"] += size sums["bits"] += total for sums in result.values(): sums["bits_per_byte"] = sums["bits"] / sums["bytes"] data = dict(weights=ident, seed=4419073, prompt_prefix=prompt_prefix, summary=result, items=rows) a.out.parent.mkdir(exist_ok=True, parents=True) a.out.write_text(json.dumps(data, indent=2) + "\n") print({k: round(v["bits_per_byte"], 4) for k, v in result.items()}, flush=True) if __name__ == "__main__": main()