"""Continue a text with the web page's engine: python generate_fast.py "The problem of personal identity" Needs wasmtime, numpy and tokenizers (pip install wasmtime numpy tokenizers), and the files core.wasm, core.bin and tokenizer.json from this repository. core.wasm is the Rust engine of gustavecortal.com/philosophy-lm/, compiled to WebAssembly, and core.bin its weights: 8-bit matrices, a 4-bit bigram table (0.2% higher bits per byte than the float32 model). It runs on one thread, with the web page's sampling: presence penalty 2.0, then temperature 0.7, top-k 20 and top-p 0.8. A text is continued as a new document (the end-of-text token first), as in training.""" import argparse import time import numpy as np import wasmtime from tokenizers import Tokenizer EOS = 0 class Core: """The engine on one thread: run() takes up to `chunk` tokens at the next positions and returns the last one's logits.""" def __init__(self, wasm="core.wasm", weights="core.bin"): engine = wasmtime.Engine() self.store = wasmtime.Store(engine) module = wasmtime.Module.from_file(engine, wasm) # helper threads only: the engine calls these when set_helpers gave it helpers, never here def unused(*args): raise RuntimeError("the engine asked for a helper thread") imports = [] for item in module.imports: imports.append(wasmtime.Func(self.store, item.type, unused)) e = wasmtime.Instance(self.store, module, imports).exports(self.store) self.e, self.memory = e, e["memory"] data = open(weights, "rb").read() at = e["alloc"](self.store, len(data)) if not at: raise MemoryError("not enough memory for the weights") self.memory.write(self.store, data, at) if e["load"](self.store, at, len(data)) != 0: raise ValueError(f"{weights} could not be read") self.vocab, self.chunk, self.context = (e[k](self.store) for k in ("vocab", "chunk", "context")) def reset(self): self.e["reset"](self.store) def run(self, tokens, logits=True): self.memory.write(self.store, np.asarray(tokens, dtype=np.uint32).tobytes(), self.e["tokens"](self.store)) p = self.e["run"](self.store, len(tokens), int(logits)) if logits: return np.frombuffer(self.memory.read(self.store, p, p + 4 * self.vocab), dtype=np.float32) def prefill(self, tokens): self.reset() for at in range(0, len(tokens), self.chunk): logits = self.run(tokens[at:at + self.chunk], at + self.chunk >= len(tokens)) return logits def pick(logits, seen, rng, temperature, top_k, top_p, presence_penalty): """generate.py's sampler: presence penalty, then temperature, top-k and top-p.""" logits = logits.astype(np.float64) if seen: logits[list(seen)] -= presence_penalty top = np.argpartition(-logits, top_k)[:top_k] top = top[np.argsort(-logits[top])] probs = np.exp((logits[top] - logits[top[0]]) / temperature) probs /= probs.sum() keep = (np.cumsum(probs) - probs) < top_p probs = probs[keep] / probs[keep].sum() return int(rng.choice(top[keep], p=probs)) def main(): p = argparse.ArgumentParser() p.add_argument("text") p.add_argument("--tokens", type=int, default=256) p.add_argument("--temperature", type=float, default=0.7) p.add_argument("--top-k", type=int, default=20) p.add_argument("--top-p", type=float, default=0.8) p.add_argument("--presence-penalty", type=float, default=2.0) p.add_argument("--seed", type=int, default=None) args = p.parse_args() tokenizer = Tokenizer.from_file("tokenizer.json") core = Core() rng = np.random.default_rng(args.seed) prompt = [EOS] + tokenizer.encode(args.text, add_special_tokens=False).ids logits = core.prefill(prompt) seen, out, start = set(), [], time.perf_counter() for _ in range(min(args.tokens, core.context - len(prompt))): token = pick(logits, seen, rng, args.temperature, args.top_k, args.top_p, args.presence_penalty) if token == EOS: break seen.add(token) out.append(token) logits = core.run([token]) seconds = time.perf_counter() - start print(args.text + tokenizer.decode(out)) print(f"\n[{len(out)} tokens in {seconds:.2f} s, {len(out) / seconds:.0f} tokens/s]") if __name__ == "__main__": main()