PhiloLM / generate_fast.py
gustavecortal's picture
Card: architecture figure and short architecture section; generate_fast.py runs the web page's engine (about 150 tokens/s on one CPU thread)
63805f8 verified
Raw History Blame Contribute Delete
4.45 kB
"""Continue a text with the web page's engine: python generate_fast.py "The problem of personal identity"
Needs wasmtime, numpy and tokenizers (pip install wasmtime numpy tokenizers), and the files core.wasm, core.bin and
tokenizer.json from this repository. core.wasm is the Rust engine of gustavecortal.com/philosophy-lm/, compiled to
WebAssembly, and core.bin its weights: 8-bit matrices, a 4-bit bigram table (0.2% higher bits per byte than the
float32 model). It runs on one thread, with the web page's sampling: presence penalty 2.0, then temperature 0.7,
top-k 20 and top-p 0.8. A text is continued as a new document (the end-of-text token first), as in training."""
import argparse
import time
import numpy as np
import wasmtime
from tokenizers import Tokenizer
EOS = 0
class Core:
"""The engine on one thread: run() takes up to `chunk` tokens at the next positions and returns the last one's
logits."""
def __init__(self, wasm="core.wasm", weights="core.bin"):
engine = wasmtime.Engine()
self.store = wasmtime.Store(engine)
module = wasmtime.Module.from_file(engine, wasm)
# helper threads only: the engine calls these when set_helpers gave it helpers, never here
def unused(*args):
raise RuntimeError("the engine asked for a helper thread")
imports = []
for item in module.imports:
imports.append(wasmtime.Func(self.store, item.type, unused))
e = wasmtime.Instance(self.store, module, imports).exports(self.store)
self.e, self.memory = e, e["memory"]
data = open(weights, "rb").read()
at = e["alloc"](self.store, len(data))
if not at:
raise MemoryError("not enough memory for the weights")
self.memory.write(self.store, data, at)
if e["load"](self.store, at, len(data)) != 0:
raise ValueError(f"{weights} could not be read")
self.vocab, self.chunk, self.context = (e[k](self.store) for k in ("vocab", "chunk", "context"))
def reset(self):
self.e["reset"](self.store)
def run(self, tokens, logits=True):
self.memory.write(self.store, np.asarray(tokens, dtype=np.uint32).tobytes(), self.e["tokens"](self.store))
p = self.e["run"](self.store, len(tokens), int(logits))
if logits:
return np.frombuffer(self.memory.read(self.store, p, p + 4 * self.vocab), dtype=np.float32)
def prefill(self, tokens):
self.reset()
for at in range(0, len(tokens), self.chunk):
logits = self.run(tokens[at:at + self.chunk], at + self.chunk >= len(tokens))
return logits
def pick(logits, seen, rng, temperature, top_k, top_p, presence_penalty):
"""generate.py's sampler: presence penalty, then temperature, top-k and top-p."""
logits = logits.astype(np.float64)
if seen:
logits[list(seen)] -= presence_penalty
top = np.argpartition(-logits, top_k)[:top_k]
top = top[np.argsort(-logits[top])]
probs = np.exp((logits[top] - logits[top[0]]) / temperature)
probs /= probs.sum()
keep = (np.cumsum(probs) - probs) < top_p
probs = probs[keep] / probs[keep].sum()
return int(rng.choice(top[keep], p=probs))
def main():
p = argparse.ArgumentParser()
p.add_argument("text")
p.add_argument("--tokens", type=int, default=256)
p.add_argument("--temperature", type=float, default=0.7)
p.add_argument("--top-k", type=int, default=20)
p.add_argument("--top-p", type=float, default=0.8)
p.add_argument("--presence-penalty", type=float, default=2.0)
p.add_argument("--seed", type=int, default=None)
args = p.parse_args()
tokenizer = Tokenizer.from_file("tokenizer.json")
core = Core()
rng = np.random.default_rng(args.seed)
prompt = [EOS] + tokenizer.encode(args.text, add_special_tokens=False).ids
logits = core.prefill(prompt)
seen, out, start = set(), [], time.perf_counter()
for _ in range(min(args.tokens, core.context - len(prompt))):
token = pick(logits, seen, rng, args.temperature, args.top_k, args.top_p, args.presence_penalty)
if token == EOS:
break
seen.add(token)
out.append(token)
logits = core.run([token])
seconds = time.perf_counter() - start
print(args.text + tokenizer.decode(out))
print(f"\n[{len(out)} tokens in {seconds:.2f} s, {len(out) / seconds:.0f} tokens/s]")
if __name__ == "__main__":
main()