"""LFM3-1.7B + deliberation module: an inference-time controller of WHEN the model stops thinking. Measured problem of the base model: it often already knows the answer but keeps reasoning until the token budget runs out, and a cut-off reasoning has no answer at all. The module watches the reasoning while it is generated: * answer read-out probe: every `every` tokens of thinking, on a separate server slot, the reasoning so far is closed at the last sentence end exactly the way the model closes it itself ("..." + ) and the model gives a short forced answer greedily. The main reasoning stream is not touched. * temporal consensus (multiple-choice only): if `agree` successive probes give the same answer, the reasoning is committed there (stream dropped, that answer returned). On dev it helped multiple-choice questions and hurt numeric math at a 16k budget (the model revises "stable" intermediate results late), so it is off for math. * budget fallback (all kinds): if the reasoning reaches the budget, it is closed and the model answers instead of being cut off: a short forced answer for math / choice questions; for free-form answers (code, writing) the thinking is closed `reserve` tokens before the budget and the answer is generated with the normal sampler. * test-and-repair (code, `solve_code` / --kind code): the program is run on the public tests written in the statement; on a failure the model is shown expected vs actual output and fixes the program (up to 2 rounds). * render-check-repair (visual, `solve_visual` / --kind visual): an SVG drawing or a web page (HTML, canvas, three.js) is rendered in headless Chrome; invalid SVG, JavaScript errors, failed resources, a blank page or an animation that does not move are shown to the model, which fixes the file (up to 2 rounds). Requires a running llama-server with the model (e.g. gguf/lfm3-orig-q5km.gguf) and at least 2 slots (-np 2). Usage: python lfm3_deliberate.py --kind math "How many positive divisors does 2024 have?" python lfm3_deliberate.py --kind choice "Which planet ... ?\nA. ...\nB. ...\nAnswer with 'The answer is (X)'." python lfm3_deliberate.py --kind free --budget 16384 "Write a Python function that ..." python lfm3_deliberate.py --kind code " Write a Python 3 program ..." python lfm3_deliberate.py --kind visual "Draw a compass rose ... Answer with one complete SVG document ..." """ import argparse import json import queue import re import threading import urllib.request from transformers import AutoTokenizer THINK_END, EOS = 124902, 124900 def _runnable(text): """True if the answer contains a code block that compiles as Python.""" blocks = re.findall(r"```(?:python|py)?[ \t]*\n(.*?)```", text or "", re.S) if not blocks: return False try: compile(blocks[-1], "", "exec") return True except Exception: return False ANIMATED = re.compile(r"animat|rotat|spinn|bounc|orbit|\bmoving\b|\bflying\b|\bfalling\b|scroll|\bgame\b", re.I) PROBES = {"math": ("The final answer is $\\boxed{", 48, ["}$", "}\n"], "}$"), "choice": ("The answer is (", 3, [")"], ")."), "phrase": ("The answer is ", 24, ["\n", ".\n"], ".")} class Deliberator: def __init__(self, url="http://localhost:8080", tokenizer="tokenizer", budget=16384, every=2048, agree=3, reserve=4096, temperature=0.6, top_p=0.95, top_k=50, main_slot=0, probe_slot=1, code_rescue=False): self.url, self.budget, self.every, self.agree, self.reserve = url + "/completion", budget, every, agree, reserve self.code_rescue = code_rescue self.sampling = {"temperature": temperature, "top_p": top_p, "top_k": top_k, "min_p": 0.0, "repeat_penalty": 1.0, "samplers": ["top_k", "top_p", "temperature"]} self.main_slot, self.probe_slot = main_slot, probe_slot self.tok = AutoTokenizer.from_pretrained(tokenizer) self.ends = set() for i in range(len(self.tok)): t = self.tok.convert_ids_to_tokens(i) if t is not None and (t.endswith(".") or t.endswith("Ċ")): self.ends.add(i) self.probe_ids = {k: self.tok(v[0], add_special_tokens=False)["input_ids"] for k, v in PROBES.items()} def _post(self, body): req = urllib.request.Request(self.url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) return json.loads(urllib.request.urlopen(req, timeout=3600).read()) def _snap(self, gen, k, lo=0): for j in range(min(k, len(gen)), lo, -1): if gen[j - 1] in self.ends: return j return None def _probe(self, ids, gen, cut, kind): suf, n_new, stops, close = PROBES[kind] pr = self._post({"prompt": ids + gen[:cut] + [THINK_END] + self.probe_ids[kind], "n_predict": n_new, "temperature": 0.0, "cache_prompt": False, "id_slot": self.probe_slot, "stop": stops}) c = pr["content"] if close == "}$": depth = 1 for j, ch in enumerate(c): depth += (ch == "{") - (ch == "}") if depth == 0: c = c[:j] break c = c.strip().rstrip(".").strip() if close == "." else c.strip() ans = re.sub(r"\s+", "", c).lower() if c and "..." not in c and len(c) <= 80 else None return suf + c + close, ans def _ids(self, question): msgs = question if isinstance(question, list) else [{"role": "user", "content": question}] return self.tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False) def _readout(self, ids, gen, cut, reserve, seed, code=False): """Answer after thinking closed at `cut`. For code: if that answer has no program that compiles (the model tends to write an editorial with pseudo-code at the budget), answer again with ```python already opened.""" pr = self._post({"prompt": ids + gen[:cut] + [THINK_END], "n_predict": reserve, "seed": seed + 17, "cache_prompt": False, "id_slot": self.probe_slot, **self.sampling}) text = pr["content"] if code and self.code_rescue and not _runnable(text): pre = self.tok("```python\n", add_special_tokens=False)["input_ids"] pr = self._post({"prompt": ids + gen[:cut] + [THINK_END] + pre, "n_predict": reserve, "seed": seed + 29, "cache_prompt": False, "id_slot": self.probe_slot, **self.sampling}) text = "```python\n" + pr["content"] if "```" in pr["content"] else "```python\n" + pr["content"].rstrip() + "\n```" return text def ask(self, question, kind="math", seed=0, code=False): """question: a string or a chat message list. kind: 'math' (boxed answer), 'choice' (letter), 'phrase' (short phrase), 'free' (code / text); code=True marks a programming task (code read-out rescue at the budget). Returns dict(answer, thinking_tokens, mode) with mode in {natural, commit, fallback}.""" ids = self._ids(question) body = {"prompt": ids, "n_predict": self.budget, "seed": seed, "cache_prompt": False, "stream": True, "return_tokens": True, "id_slot": self.main_slot, **self.sampling} resp = urllib.request.urlopen(urllib.request.Request(self.url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}), timeout=3600) commit_allowed = kind in ("choice", "phrase") gen, closed, stop, next_probe, prev_cut = [], False, None, self.every, 0 jobs, results = queue.Queue(), [] def prober(): while (job := jobs.get()) is not None: results.append((job, *self._probe(ids, job[1], job[0], kind))) th = threading.Thread(target=prober, daemon=True) th.start() commit = None while True: line = resp.readline() if not line: break line = line.decode("utf-8").strip() if not line.startswith("data:"): continue ev = json.loads(line[5:]) gen += ev.get("tokens", []) if THINK_END in ev.get("tokens", []): closed = True if ev.get("stop"): stop = ev.get("stop_type") break if commit_allowed and not closed and len(gen) >= next_probe: next_probe += self.every cut = self._snap(gen, len(gen), prev_cut) if cut: prev_cut = cut jobs.put((cut, list(gen[:cut]))) answers = [a for _, _, a in results] if len(answers) >= self.agree and answers[-1] is not None and len(set(answers[-self.agree:])) == 1: job, text, _ = results[-1] commit = (job[0], text) resp.close() break jobs.put(None) if gen and gen[-1] == EOS: gen = gen[:-1] if commit: cut, text = commit return {"answer": text, "thinking_tokens": cut, "mode": "commit"} if stop == "limit" and not closed: # budget exhausted while thinking: answer instead of being cut off if kind == "free": cut = self._snap(gen, self.budget - self.reserve) or (self.budget - self.reserve) return {"answer": self._readout(ids, gen, cut, self.reserve, seed, code), "thinking_tokens": cut, "mode": "fallback"} cut = self._snap(gen, self.budget - 64) or (self.budget - 64) text, _ = self._probe(ids, gen, cut, kind) return {"answer": text, "thinking_tokens": cut, "mode": "fallback"} text = self.tok.decode(gen) answer = text.split("", 1)[1].strip() if "" in text else text return {"answer": answer, "thinking_tokens": len(gen), "mode": "natural"} def ask_parallel(self, question, kind="math", k=4, seed=0, slots=None): """Parallel thinking: k independent reasonings (each with the deliberation module, on slots[2i] / slots[2i+1], by default 0..2k-1, so the server needs -np >= 2k), final answer by majority vote over their extracted answers. On AIME (test, 4 samples) a plain majority lifted accuracy from 52.5 to 65.0 at 4x the tokens. Returns dict(answer, votes, outputs).""" slots = slots or list(range(2 * k)) import collections import copy import os import sys from concurrent.futures import ThreadPoolExecutor sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime")) from grade import last_boxed def one(i): d = copy.copy(self) d.main_slot, d.probe_slot = slots[2 * i], slots[2 * i + 1] return d.ask(question, kind, seed=seed + 1000 * i) def key(text): if kind == "math": b = last_boxed(text) return re.sub(r"\s+", "", b) if b else None if kind == "choice": m = re.findall(r"answer is:?\s*\(?([A-J])\b", text) return m[-1] if m else None m = re.findall(r"answer is:?\s*(.+)", text, re.I) return re.sub(r"\s+", " ", m[-1]).strip().rstrip(".").lower() if m else None with ThreadPoolExecutor(k) as ex: outs = list(ex.map(one, range(k))) keys = [key(o["answer"]) for o in outs] # weighted vote (rule chosen on dev math, applied once to AIME test: 66.7 vs 65.0 plain majority): # an answer the model reached by itself counts 4x an answer read out at the budget, and shorter # reasoning counts more (weight ~ 1/sqrt(thinking tokens)) votes = collections.defaultdict(float) for k_, o in zip(keys, outs): if k_: votes[k_] += (0.25 if o["mode"] == "fallback" else 1.0) / max(o["thinking_tokens"], 200) ** 0.5 if not votes: return {"answer": outs[0]["answer"], "votes": {}, "outputs": outs} win = max(votes, key=votes.get) return {"answer": outs[keys.index(win)]["answer"], "votes": {k_: round(v, 4) for k_, v in votes.items()}, "outputs": outs} def solve_code(self, question, rounds=2, repair_think=6144, repair_reserve=2048, seed=0): """Code task: answer with the deliberation module, then run the program on the public tests written in the statement (examples / doctests / quoted asserts). On a failure the model sees expected vs actual output and thinks again; a repaired program is kept only if it passes more public tests than the previous best. Returns dict(answer, public=[passed, total], rounds, mode).""" import sys, os sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime")) from grade import extract_code from public_tests import run_public msgs = list(question) if isinstance(question, list) else [{"role": "user", "content": question}] question = next(m["content"] for m in reversed(msgs) if m["role"] == "user") kind = "humaneval" if ">>>" in question else "mbpp" if "assert " in question else "code_io" first = self.ask(msgs, "free", seed=seed, code=True) best, final, used_rounds = None, first["answer"], 0 for rnd in range(rounds + 1): code = extract_code(final) ok, total, fail = run_public(kind, code, question) if code else (0, -1, "No ```python code block found.") if best is None or ok > best[0]: best = (ok, final) if total == 0 or ok == total or rnd == rounds: break msgs += [{"role": "assistant", "content": final}, {"role": "user", "content": "I ran your program on the examples from the problem statement and " f"it does not give the expected result.\n\n{fail}\n\nFind the mistake and give the complete " "corrected program in a single ```python code block."}] ids = self.tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False) r = self._post({"prompt": ids, "n_predict": repair_think + repair_reserve, "seed": seed + 5000 + rnd, "cache_prompt": False, "return_tokens": True, "id_slot": self.probe_slot, **self.sampling}) gen = [t for t in r["tokens"] if t != EOS] if THINK_END in gen: final = r["content"].split("", 1)[1] else: # the repair turn hit its thinking budget: answer at the budget, as in scripts/repair.py lim = min(repair_think, len(gen)) cut = self._snap(gen, lim) or lim final = self._readout(ids, gen, cut, repair_reserve, seed + 5000 + rnd, code=True) used_rounds += 1 return {"answer": best[1], "public": [best[0], total], "rounds": used_rounds, "mode": first["mode"]} def solve_visual(self, question, kind=None, animated=None, rounds=2, hints=False, repair_think=6144, repair_reserve=4096, seed=0, renderer=None): """Visual task (an SVG drawing, a web page, a canvas animation or a three.js scene): answer with the deliberation module, insert the three.js import map if a page imports 'three' without one, render the answer in headless Chrome (scripts/render.py) and repair what the browser reports (invalid SVG, JavaScript errors, failed resources, a blank page, an animation that does not move, horizontal overflow); a repair is kept only if it has fewer problems. kind: 'svg' or 'html' (detected from the request when None). Returns dict(answer, problems (0 = renders cleanly), rounds, mode).""" import os import sys import tempfile sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime")) from render import Renderer, extract from visual_loop import feedback, fix_importmap, problems msgs = list(question) if isinstance(question, list) else [{"role": "user", "content": question}] q = next(m["content"] for m in reversed(msgs) if m["role"] == "user") kind = kind or ("svg" if re.search(r"\bsvg\b", q, re.I) and not re.search(r"\bhtml\b", q, re.I) else "html") animated = bool(ANIMATED.search(q)) if animated is None else animated own = renderer is None renderer = renderer or Renderer() png = os.path.join(tempfile.gettempdir(), f"lfm3_visual_{os.getpid()}_{threading.get_ident()}.png") def check(text): src = extract(text, kind) if src and kind == "html" and fix_importmap(src) != src: text, src = text.replace(src, fix_importmap(src)), fix_importmap(src) rep = renderer.render(kind, src, png) if src else None return (text,) + problems(rep, animated, src, kind, hints) try: first = self.ask(msgs, "free", seed=seed) final, bad, lines = check(first["answer"]) best, used = (bad, final), 0 for rnd in range(rounds): if bad == 0: break msgs = msgs + [{"role": "assistant", "content": final}, {"role": "user", "content": feedback(kind, lines)}] ids = self.tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False) r = self._post({"prompt": ids, "n_predict": repair_think + repair_reserve, "seed": seed + 7000 + rnd, "cache_prompt": False, "return_tokens": True, "id_slot": self.probe_slot, **self.sampling}) gen = [t for t in r["tokens"] if t != EOS] if THINK_END in gen: out = r["content"].split("", 1)[1] else: # the repair turn hit its thinking budget: answer at the budget lim = min(repair_think, len(gen)) out = self._readout(ids, gen, self._snap(gen, lim) or lim, repair_reserve, seed + 7000 + rnd) final, bad, lines = check(out) used += 1 if bad < best[0]: best = (bad, final) return {"answer": best[1], "problems": best[0], "rounds": used, "mode": first["mode"]} finally: if own: renderer.close() if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("question") ap.add_argument("--kind", default="math", choices=["math", "choice", "phrase", "free", "code", "visual"]) ap.add_argument("--budget", type=int, default=16384) ap.add_argument("--parallel", type=int, default=1, help="k parallel reasonings + majority vote (server -np >= 2k)") ap.add_argument("--url", default="http://localhost:8080") ap.add_argument("--tokenizer", default="tokenizer") a = ap.parse_args() d = Deliberator(a.url, a.tokenizer, budget=a.budget) if a.kind == "code": r = d.solve_code(a.question) print(f"[{r['mode']}, public tests {r['public'][0]}/{r['public'][1]}, {r['rounds']} repair rounds]\n{r['answer']}") elif a.kind == "visual": r = d.solve_visual(a.question) print(f"[{r['mode']}, render problems {r['problems']}, {r['rounds']} repair rounds]\n{r['answer']}") elif a.parallel > 1: r = d.ask_parallel(a.question, a.kind, k=a.parallel) print(f"[votes {r['votes']}]\n{r['answer']}") else: r = d.ask(a.question, a.kind) print(f"[{r['mode']}, {r['thinking_tokens']} thinking tokens]\n{r['answer']}")