HelioAI's picture
LFM3-1.7B inference modules: experimental release
67ebcef verified
Raw History Blame Contribute Delete
20 kB
"""LFM3-1.7B + deliberation module: an inference-time controller of WHEN the model stops thinking.
Measured problem of the base model: it often already knows the answer but keeps reasoning until the token budget
runs out, and a cut-off reasoning has no answer at all. The module watches the reasoning while it is generated:
* answer read-out probe: every `every` tokens of thinking, on a separate server slot, the reasoning so far is closed
at the last sentence end exactly the way the model closes it itself ("..." + </think>) and the model gives a short
forced answer greedily. The main reasoning stream is not touched.
* temporal consensus (multiple-choice only): if `agree` successive probes give the same answer, the reasoning is
committed there (stream dropped, that answer returned). On dev it helped multiple-choice questions and hurt
numeric math at a 16k budget (the model revises "stable" intermediate results late), so it is off for math.
* budget fallback (all kinds): if the reasoning reaches the budget, it is closed and the model answers instead of
being cut off: a short forced answer for math / choice questions; for free-form answers (code, writing) the
thinking is closed `reserve` tokens before the budget and the answer is generated with the normal sampler.
* test-and-repair (code, `solve_code` / --kind code): the program is run on the public tests written in the
statement; on a failure the model is shown expected vs actual output and fixes the program (up to 2 rounds).
* render-check-repair (visual, `solve_visual` / --kind visual): an SVG drawing or a web page (HTML, canvas,
three.js) is rendered in headless Chrome; invalid SVG, JavaScript errors, failed resources, a blank page or an
animation that does not move are shown to the model, which fixes the file (up to 2 rounds).
Requires a running llama-server with the model (e.g. gguf/lfm3-orig-q5km.gguf) and at least 2 slots (-np 2).
Usage:
python lfm3_deliberate.py --kind math "How many positive divisors does 2024 have?"
python lfm3_deliberate.py --kind choice "Which planet ... ?\nA. ...\nB. ...\nAnswer with 'The answer is (X)'."
python lfm3_deliberate.py --kind free --budget 16384 "Write a Python function that ..."
python lfm3_deliberate.py --kind code "<statement with Input/Output examples> Write a Python 3 program ..."
python lfm3_deliberate.py --kind visual "Draw a compass rose ... Answer with one complete SVG document ..."
"""
import argparse
import json
import queue
import re
import threading
import urllib.request
from transformers import AutoTokenizer
THINK_END, EOS = 124902, 124900
def _runnable(text):
"""True if the answer contains a code block that compiles as Python."""
blocks = re.findall(r"```(?:python|py)?[ \t]*\n(.*?)```", text or "", re.S)
if not blocks:
return False
try:
compile(blocks[-1], "<answer>", "exec")
return True
except Exception:
return False
ANIMATED = re.compile(r"animat|rotat|spinn|bounc|orbit|\bmoving\b|\bflying\b|\bfalling\b|scroll|\bgame\b", re.I)
PROBES = {"math": ("The final answer is $\\boxed{", 48, ["}$", "}\n"], "}$"),
"choice": ("The answer is (", 3, [")"], ")."),
"phrase": ("The answer is ", 24, ["\n", ".\n"], ".")}
class Deliberator:
def __init__(self, url="http://localhost:8080", tokenizer="tokenizer", budget=16384, every=2048, agree=3,
reserve=4096, temperature=0.6, top_p=0.95, top_k=50, main_slot=0, probe_slot=1, code_rescue=False):
self.url, self.budget, self.every, self.agree, self.reserve = url + "/completion", budget, every, agree, reserve
self.code_rescue = code_rescue
self.sampling = {"temperature": temperature, "top_p": top_p, "top_k": top_k, "min_p": 0.0,
"repeat_penalty": 1.0, "samplers": ["top_k", "top_p", "temperature"]}
self.main_slot, self.probe_slot = main_slot, probe_slot
self.tok = AutoTokenizer.from_pretrained(tokenizer)
self.ends = set()
for i in range(len(self.tok)):
t = self.tok.convert_ids_to_tokens(i)
if t is not None and (t.endswith(".") or t.endswith("Ċ")):
self.ends.add(i)
self.probe_ids = {k: self.tok(v[0], add_special_tokens=False)["input_ids"] for k, v in PROBES.items()}
def _post(self, body):
req = urllib.request.Request(self.url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})
return json.loads(urllib.request.urlopen(req, timeout=3600).read())
def _snap(self, gen, k, lo=0):
for j in range(min(k, len(gen)), lo, -1):
if gen[j - 1] in self.ends:
return j
return None
def _probe(self, ids, gen, cut, kind):
suf, n_new, stops, close = PROBES[kind]
pr = self._post({"prompt": ids + gen[:cut] + [THINK_END] + self.probe_ids[kind], "n_predict": n_new,
"temperature": 0.0, "cache_prompt": False, "id_slot": self.probe_slot, "stop": stops})
c = pr["content"]
if close == "}$":
depth = 1
for j, ch in enumerate(c):
depth += (ch == "{") - (ch == "}")
if depth == 0:
c = c[:j]
break
c = c.strip().rstrip(".").strip() if close == "." else c.strip()
ans = re.sub(r"\s+", "", c).lower() if c and "..." not in c and len(c) <= 80 else None
return suf + c + close, ans
def _ids(self, question):
msgs = question if isinstance(question, list) else [{"role": "user", "content": question}]
return self.tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False)
def _readout(self, ids, gen, cut, reserve, seed, code=False):
"""Answer after thinking closed at `cut`. For code: if that answer has no program that compiles (the model
tends to write an editorial with pseudo-code at the budget), answer again with ```python already opened."""
pr = self._post({"prompt": ids + gen[:cut] + [THINK_END], "n_predict": reserve, "seed": seed + 17,
"cache_prompt": False, "id_slot": self.probe_slot, **self.sampling})
text = pr["content"]
if code and self.code_rescue and not _runnable(text):
pre = self.tok("```python\n", add_special_tokens=False)["input_ids"]
pr = self._post({"prompt": ids + gen[:cut] + [THINK_END] + pre, "n_predict": reserve, "seed": seed + 29,
"cache_prompt": False, "id_slot": self.probe_slot, **self.sampling})
text = "```python\n" + pr["content"] if "```" in pr["content"] else "```python\n" + pr["content"].rstrip() + "\n```"
return text
def ask(self, question, kind="math", seed=0, code=False):
"""question: a string or a chat message list. kind: 'math' (boxed answer), 'choice' (letter), 'phrase'
(short phrase), 'free' (code / text); code=True marks a programming task (code read-out rescue at the budget).
Returns dict(answer, thinking_tokens, mode) with mode in {natural, commit, fallback}."""
ids = self._ids(question)
body = {"prompt": ids, "n_predict": self.budget, "seed": seed, "cache_prompt": False, "stream": True,
"return_tokens": True, "id_slot": self.main_slot, **self.sampling}
resp = urllib.request.urlopen(urllib.request.Request(self.url, data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"}),
timeout=3600)
commit_allowed = kind in ("choice", "phrase")
gen, closed, stop, next_probe, prev_cut = [], False, None, self.every, 0
jobs, results = queue.Queue(), []
def prober():
while (job := jobs.get()) is not None:
results.append((job, *self._probe(ids, job[1], job[0], kind)))
th = threading.Thread(target=prober, daemon=True)
th.start()
commit = None
while True:
line = resp.readline()
if not line:
break
line = line.decode("utf-8").strip()
if not line.startswith("data:"):
continue
ev = json.loads(line[5:])
gen += ev.get("tokens", [])
if THINK_END in ev.get("tokens", []):
closed = True
if ev.get("stop"):
stop = ev.get("stop_type")
break
if commit_allowed and not closed and len(gen) >= next_probe:
next_probe += self.every
cut = self._snap(gen, len(gen), prev_cut)
if cut:
prev_cut = cut
jobs.put((cut, list(gen[:cut])))
answers = [a for _, _, a in results]
if len(answers) >= self.agree and answers[-1] is not None and len(set(answers[-self.agree:])) == 1:
job, text, _ = results[-1]
commit = (job[0], text)
resp.close()
break
jobs.put(None)
if gen and gen[-1] == EOS:
gen = gen[:-1]
if commit:
cut, text = commit
return {"answer": text, "thinking_tokens": cut, "mode": "commit"}
if stop == "limit" and not closed: # budget exhausted while thinking: answer instead of being cut off
if kind == "free":
cut = self._snap(gen, self.budget - self.reserve) or (self.budget - self.reserve)
return {"answer": self._readout(ids, gen, cut, self.reserve, seed, code), "thinking_tokens": cut,
"mode": "fallback"}
cut = self._snap(gen, self.budget - 64) or (self.budget - 64)
text, _ = self._probe(ids, gen, cut, kind)
return {"answer": text, "thinking_tokens": cut, "mode": "fallback"}
text = self.tok.decode(gen)
answer = text.split("</think>", 1)[1].strip() if "</think>" in text else text
return {"answer": answer, "thinking_tokens": len(gen), "mode": "natural"}
def ask_parallel(self, question, kind="math", k=4, seed=0, slots=None):
"""Parallel thinking: k independent reasonings (each with the deliberation module, on slots[2i] / slots[2i+1],
by default 0..2k-1, so the server needs -np >= 2k), final answer by majority vote over their extracted
answers. On AIME (test, 4 samples) a plain majority lifted accuracy from 52.5 to 65.0 at 4x the tokens.
Returns dict(answer, votes, outputs)."""
slots = slots or list(range(2 * k))
import collections
import copy
import os
import sys
from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime"))
from grade import last_boxed
def one(i):
d = copy.copy(self)
d.main_slot, d.probe_slot = slots[2 * i], slots[2 * i + 1]
return d.ask(question, kind, seed=seed + 1000 * i)
def key(text):
if kind == "math":
b = last_boxed(text)
return re.sub(r"\s+", "", b) if b else None
if kind == "choice":
m = re.findall(r"answer is:?\s*\(?([A-J])\b", text)
return m[-1] if m else None
m = re.findall(r"answer is:?\s*(.+)", text, re.I)
return re.sub(r"\s+", " ", m[-1]).strip().rstrip(".").lower() if m else None
with ThreadPoolExecutor(k) as ex:
outs = list(ex.map(one, range(k)))
keys = [key(o["answer"]) for o in outs]
# weighted vote (rule chosen on dev math, applied once to AIME test: 66.7 vs 65.0 plain majority):
# an answer the model reached by itself counts 4x an answer read out at the budget, and shorter
# reasoning counts more (weight ~ 1/sqrt(thinking tokens))
votes = collections.defaultdict(float)
for k_, o in zip(keys, outs):
if k_:
votes[k_] += (0.25 if o["mode"] == "fallback" else 1.0) / max(o["thinking_tokens"], 200) ** 0.5
if not votes:
return {"answer": outs[0]["answer"], "votes": {}, "outputs": outs}
win = max(votes, key=votes.get)
return {"answer": outs[keys.index(win)]["answer"], "votes": {k_: round(v, 4) for k_, v in votes.items()},
"outputs": outs}
def solve_code(self, question, rounds=2, repair_think=6144, repair_reserve=2048, seed=0):
"""Code task: answer with the deliberation module, then run the program on the public tests written in the
statement (examples / doctests / quoted asserts). On a failure the model sees expected vs actual output and
thinks again; a repaired program is kept only if it passes more public tests than the previous best.
Returns dict(answer, public=[passed, total], rounds, mode)."""
import sys, os
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime"))
from grade import extract_code
from public_tests import run_public
msgs = list(question) if isinstance(question, list) else [{"role": "user", "content": question}]
question = next(m["content"] for m in reversed(msgs) if m["role"] == "user")
kind = "humaneval" if ">>>" in question else "mbpp" if "assert " in question else "code_io"
first = self.ask(msgs, "free", seed=seed, code=True)
best, final, used_rounds = None, first["answer"], 0
for rnd in range(rounds + 1):
code = extract_code(final)
ok, total, fail = run_public(kind, code, question) if code else (0, -1, "No ```python code block found.")
if best is None or ok > best[0]:
best = (ok, final)
if total == 0 or ok == total or rnd == rounds:
break
msgs += [{"role": "assistant", "content": final},
{"role": "user", "content": "I ran your program on the examples from the problem statement and "
f"it does not give the expected result.\n\n{fail}\n\nFind the mistake and give the complete "
"corrected program in a single ```python code block."}]
ids = self.tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False)
r = self._post({"prompt": ids, "n_predict": repair_think + repair_reserve, "seed": seed + 5000 + rnd,
"cache_prompt": False, "return_tokens": True, "id_slot": self.probe_slot, **self.sampling})
gen = [t for t in r["tokens"] if t != EOS]
if THINK_END in gen:
final = r["content"].split("</think>", 1)[1]
else: # the repair turn hit its thinking budget: answer at the budget, as in scripts/repair.py
lim = min(repair_think, len(gen))
cut = self._snap(gen, lim) or lim
final = self._readout(ids, gen, cut, repair_reserve, seed + 5000 + rnd, code=True)
used_rounds += 1
return {"answer": best[1], "public": [best[0], total], "rounds": used_rounds, "mode": first["mode"]}
def solve_visual(self, question, kind=None, animated=None, rounds=2, hints=False, repair_think=6144,
repair_reserve=4096, seed=0, renderer=None):
"""Visual task (an SVG drawing, a web page, a canvas animation or a three.js scene): answer with the
deliberation module, insert the three.js import map if a page imports 'three' without one, render the
answer in headless Chrome (scripts/render.py) and repair what the browser reports (invalid SVG, JavaScript
errors, failed resources, a blank page, an animation that does not move, horizontal overflow); a repair is
kept only if it has fewer problems. kind: 'svg' or 'html' (detected from the request when None).
Returns dict(answer, problems (0 = renders cleanly), rounds, mode)."""
import os
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime"))
from render import Renderer, extract
from visual_loop import feedback, fix_importmap, problems
msgs = list(question) if isinstance(question, list) else [{"role": "user", "content": question}]
q = next(m["content"] for m in reversed(msgs) if m["role"] == "user")
kind = kind or ("svg" if re.search(r"\bsvg\b", q, re.I) and not re.search(r"\bhtml\b", q, re.I) else "html")
animated = bool(ANIMATED.search(q)) if animated is None else animated
own = renderer is None
renderer = renderer or Renderer()
png = os.path.join(tempfile.gettempdir(), f"lfm3_visual_{os.getpid()}_{threading.get_ident()}.png")
def check(text):
src = extract(text, kind)
if src and kind == "html" and fix_importmap(src) != src:
text, src = text.replace(src, fix_importmap(src)), fix_importmap(src)
rep = renderer.render(kind, src, png) if src else None
return (text,) + problems(rep, animated, src, kind, hints)
try:
first = self.ask(msgs, "free", seed=seed)
final, bad, lines = check(first["answer"])
best, used = (bad, final), 0
for rnd in range(rounds):
if bad == 0:
break
msgs = msgs + [{"role": "assistant", "content": final}, {"role": "user", "content": feedback(kind, lines)}]
ids = self.tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False)
r = self._post({"prompt": ids, "n_predict": repair_think + repair_reserve, "seed": seed + 7000 + rnd,
"cache_prompt": False, "return_tokens": True, "id_slot": self.probe_slot,
**self.sampling})
gen = [t for t in r["tokens"] if t != EOS]
if THINK_END in gen:
out = r["content"].split("</think>", 1)[1]
else: # the repair turn hit its thinking budget: answer at the budget
lim = min(repair_think, len(gen))
out = self._readout(ids, gen, self._snap(gen, lim) or lim, repair_reserve, seed + 7000 + rnd)
final, bad, lines = check(out)
used += 1
if bad < best[0]:
best = (bad, final)
return {"answer": best[1], "problems": best[0], "rounds": used, "mode": first["mode"]}
finally:
if own:
renderer.close()
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("question")
ap.add_argument("--kind", default="math", choices=["math", "choice", "phrase", "free", "code", "visual"])
ap.add_argument("--budget", type=int, default=16384)
ap.add_argument("--parallel", type=int, default=1, help="k parallel reasonings + majority vote (server -np >= 2k)")
ap.add_argument("--url", default="http://localhost:8080")
ap.add_argument("--tokenizer", default="tokenizer")
a = ap.parse_args()
d = Deliberator(a.url, a.tokenizer, budget=a.budget)
if a.kind == "code":
r = d.solve_code(a.question)
print(f"[{r['mode']}, public tests {r['public'][0]}/{r['public'][1]}, {r['rounds']} repair rounds]\n{r['answer']}")
elif a.kind == "visual":
r = d.solve_visual(a.question)
print(f"[{r['mode']}, render problems {r['problems']}, {r['rounds']} repair rounds]\n{r['answer']}")
elif a.parallel > 1:
r = d.ask_parallel(a.question, a.kind, k=a.parallel)
print(f"[votes {r['votes']}]\n{r['answer']}")
else:
r = d.ask(a.question, a.kind)
print(f"[{r['mode']}, {r['thinking_tokens']} thinking tokens]\n{r['answer']}")