Download source/tools/byte_fluency.py from yunusemrejr/Franok: direct link, hf CLI and curl.
- Browser
- Download file 6.41 kB
-
https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tools/byte_fluency.py
- Command line
-
hf download hf://yunusemrejr/Franok/source/tools/byte_fluency.py
-
curl -L -o byte_fluency.py https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tools/byte_fluency.py
6.41 kB
| """Measure next-text bits per UTF-8 byte on exactly the same strings with either vocabulary.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import math | |
| from pathlib import Path | |
| import re | |
| import numpy as np | |
| import torch | |
| from torch.nn import functional as F | |
| from tokenizers import Tokenizer, decoders | |
| from safetensors.torch import load_file | |
| from franok.model import Config, Franok | |
| from tools.reading import PEERS | |
| def sample(data, tokenizer): | |
| rng = np.random.default_rng(4419073) | |
| for lang in ("en", "tr"): | |
| for name, pattern in ( | |
| ("web", r"val-\d+\.bin"), | |
| ("web-more", r"val-web-more-\d+\.bin"), | |
| ("stories", r"val-stories-\d+\.bin"), | |
| ): | |
| files = sorted( | |
| f | |
| for f in (data / lang).glob("*.bin") | |
| if re.fullmatch(pattern, f.name) and f.stat().st_size > 1100 | |
| ) | |
| if len(files) < 2: | |
| raise ValueError("Need several held-out shards for " + name) | |
| # The first shard is used by the training-time checkpoint selector. | |
| files = files[1:] | |
| seen = set() | |
| attempts = 0 | |
| while len(seen) < 32: | |
| attempts += 1 | |
| if attempts > 10000: | |
| raise ValueError("Insufficient intact source passages") | |
| file = files[rng.integers(len(files))] | |
| ids = np.memmap(file, dtype="<u2", mode="r") | |
| at = int(rng.integers(len(ids) - 512)) | |
| values = ids[at : at + 512].astype(int) | |
| chunks = np.split(values, np.flatnonzero(values < 7)) | |
| segment = max(chunks, key=len) | |
| segment = segment[segment >= 7] | |
| text = tokenizer.decode(segment.tolist()) | |
| words = list(re.finditer(r"\S+", text)) | |
| if len(words) < 60: | |
| continue | |
| lo, edge, end = words[1].start(), words[18].end(), words[58].end() | |
| prefix, suffix = text[lo:edge], text[edge:end] | |
| digest = hashlib.sha256((prefix + suffix).encode()).hexdigest() | |
| if ( | |
| digest in seen | |
| or "�" in prefix + suffix | |
| or len(suffix.encode()) > 900 | |
| ): | |
| continue | |
| seen.add(digest) | |
| yield ( | |
| lang, | |
| name, | |
| prefix, | |
| suffix, | |
| dict(file=file.name, offset=at, sha256=digest), | |
| ) | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| g = p.add_mutually_exclusive_group(required=True) | |
| g.add_argument("--run", type=Path) | |
| g.add_argument("--peer", nargs="?", const="smollm2", choices=PEERS) | |
| p.add_argument("--data", type=Path, required=True) | |
| p.add_argument("--out", type=Path, required=True) | |
| p.add_argument("--threads", type=int, default=3) | |
| a = p.parse_args() | |
| torch.set_num_threads(a.threads) | |
| franok_tokenizer = Tokenizer.from_file("web/tokenizer.json") | |
| franok_tokenizer.decoder = decoders.ByteLevel() | |
| prompt_prefix = [] | |
| if a.peer: | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| repo, revision = PEERS[a.peer] | |
| tk = AutoTokenizer.from_pretrained( | |
| repo, revision=revision, local_files_only=True | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| repo, revision=revision, local_files_only=True, dtype=torch.float32 | |
| ).eval() | |
| ident = revision | |
| if tk.encode("", add_special_tokens=True) == [tk.bos_token_id]: | |
| prompt_prefix = [tk.bos_token_id] | |
| def encode(s): | |
| return tk.encode(s, add_special_tokens=False) | |
| def decode(v): | |
| return tk.decode(v, clean_up_tokenization_spaces=False) | |
| def predict(x): | |
| return model(x, use_cache=False).logits | |
| else: | |
| model = Franok(Config(**json.loads((a.run / "config.json").read_text()))).eval() | |
| file = a.run / "model.safetensors" | |
| with file.open("rb") as f: | |
| ident = hashlib.file_digest(f, "sha256").hexdigest() | |
| model.load_state_dict(load_file(str(file))) | |
| def encode(s): | |
| return franok_tokenizer.encode(s, add_special_tokens=False).ids | |
| def decode(v): | |
| return franok_tokenizer.decode(v) | |
| predict = model | |
| result = {} | |
| rows = [] | |
| for lang, name, pre, suffix, source in sample(a.data, franok_tokenizer): | |
| head = encode(pre) | |
| allids = encode(pre + suffix) | |
| split_at_boundary = allids[: len(head)] != head | |
| if split_at_boundary: | |
| # Some vocabularies merge punctuation and a following newline. The | |
| # prefix is already visible at prediction time, so keep it intact. | |
| allids = head + encode(suffix) | |
| if not head or decode(head) != pre or decode(allids) != pre + suffix: | |
| raise RuntimeError( | |
| "The two text spans must tokenize and decode exactly: " | |
| + source["sha256"] | |
| ) | |
| context = prompt_prefix + allids[:-1] | |
| scores = predict(torch.tensor([context]))[0, len(prompt_prefix) + len(head) - 1 :].float() | |
| total = F.cross_entropy( | |
| scores, torch.tensor(allids[len(head) :]), reduction="sum" | |
| ).item() / math.log(2) | |
| size = len(suffix.encode()) | |
| rows.append( | |
| dict( | |
| lang=lang, | |
| collection=name, | |
| **source, | |
| bytes=size, | |
| bits=total, | |
| tokens=len(allids) - len(head), | |
| split_at_boundary=split_at_boundary, | |
| ) | |
| ) | |
| for key in (lang, lang + "-" + name): | |
| sums = result.setdefault(key, dict(windows=0, bytes=0, bits=0)) | |
| sums["windows"] += 1 | |
| sums["bytes"] += size | |
| sums["bits"] += total | |
| for sums in result.values(): | |
| sums["bits_per_byte"] = sums["bits"] / sums["bytes"] | |
| data = dict(weights=ident, seed=4419073, prompt_prefix=prompt_prefix, summary=result, items=rows) | |
| a.out.parent.mkdir(exist_ok=True, parents=True) | |
| a.out.write_text(json.dumps(data, indent=2) + "\n") | |
| print({k: round(v["bits_per_byte"], 4) for k, v in result.items()}, flush=True) | |
| if __name__ == "__main__": | |
| main() | |