"""Toolkit module, last step: make every answer a self-contained file and repair what fails.
For each record (answers to the toolkit prompts, after the read-out module): the toolkit is inlined or the svgkit
program is run (scripts/kit_inline.py), the result is rendered in headless Chrome, and problems (a failing program,
invalid SVG, JavaScript errors, a blank page, a static animation, overflow) go back to the model in a new turn of
the same conversation (up to --rounds); a repair is kept only if it has fewer problems. The output completions hold
the final self-contained ```svg / ```html answer, so visual_eval.py scores them like any other run.
usage: kit_repair.py --src results/vis_kit_dm/vis.jsonl --tasks data/visual/tasks_kit.jsonl --out results/vis_kitr/vis.jsonl
"""
import argparse
import json
import os
import sys
import threading
import time
import urllib.request
import zlib
from concurrent.futures import ThreadPoolExecutor
from transformers import AutoTokenizer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from kit_inline import finalize # noqa: E402
from render import Renderer # noqa: E402
from visual_loop import problems # noqa: E402
THINK_END, EOS = 124902, 124900
ap = argparse.ArgumentParser()
ap.add_argument("--src", required=True)
ap.add_argument("--tasks", required=True, help="the toolkit prompts the answers were generated from")
ap.add_argument("--out", required=True)
ap.add_argument("--rounds", type=int, default=2)
ap.add_argument("--think", type=int, default=6144)
ap.add_argument("--reserve", type=int, default=4096)
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--port", type=int, default=8080)
ap.add_argument("--hints", action="store_true")
ap.add_argument("--tokenizer", default="models/LFM3-1.7B")
args = ap.parse_args()
tok = AutoTokenizer.from_pretrained(args.tokenizer)
ENDS = {i for i in range(len(tok)) if (t := tok.convert_ids_to_tokens(i)) is not None and (t.endswith(".") or t.endswith("Ċ"))}
SAMPLING = {"temperature": 0.6, "top_p": 0.95, "top_k": 50, "min_p": 0.0, "repeat_penalty": 1.0,
"samplers": ["top_k", "top_p", "temperature"]}
tasks = {json.loads(l)["id"]: json.loads(l) for l in open(args.tasks, encoding="utf-8")}
SHOTS = os.path.join(os.path.dirname(args.out), "shots")
os.makedirs(SHOTS, exist_ok=True)
def post(body):
for attempt in range(20):
try:
p = json.loads(urllib.request.urlopen(f"http://localhost:{args.port}/props", timeout=10).read())
body = {**body, "n_predict": min(body["n_predict"], p["default_generation_settings"]["n_ctx"]
- len(body["prompt"]) - 16)}
req = urllib.request.Request(f"http://localhost:{args.port}/completion", data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"})
return json.loads(urllib.request.urlopen(req, timeout=3600).read())
except Exception:
if attempt == 19:
raise
time.sleep(10)
def generate(msgs, seed):
ids = tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False)
r = post({"prompt": ids, "n_predict": args.think + args.reserve, "seed": seed, "cache_prompt": False,
"return_tokens": True, **SAMPLING})
gen = [t for t in r["tokens"] if t != EOS]
if THINK_END in gen:
return r["content"].split("", 1)[1], r["tokens_predicted"]
lim = min(args.think, len(gen))
cut = next((j for j in range(lim, 0, -1) if gen[j - 1] in ENDS), lim)
r2 = post({"prompt": ids + gen[:cut] + [THINK_END], "n_predict": args.reserve, "seed": seed + 17,
"cache_prompt": False, **SAMPLING})
return r2["content"], cut + r2["tokens_predicted"]
local = threading.local()
def renderer():
if not hasattr(local, "r"):
local.r = Renderer()
return local.r
def check(text, task, png):
"""(final self-contained answer, badness, feedback lines)."""
final, problem, src = finalize(text, task["cat"])
kind = "svg" if task["cat"] == "svg" else "html"
if problem:
return final, 10, [problem]
rep = renderer().render(kind, src, png) if src else None
bad, lines = problems(rep, task["animated"], src, kind, args.hints)
return final, bad, lines
FEEDBACK = ("I ran your answer{how} and found these problems:\n{lines}\n\nFix them and give the complete corrected "
"{what} in a single ```{lang} code block.")
def process(rec):
task = tasks[rec["id"]]
name = f"{rec['id'].split('/')[-1]}_{rec.get('sample', 0)}"
raw = rec["completion"].split("", 1)[1] if "" in rec["completion"] else ""
final, bad, lines = check(raw, task, os.path.join(SHOTS, f"{name}_r0.png"))
first, best, used, cur = (bad, final), (bad, final, 0), 0, raw
msgs = [{"role": "user", "content": task["prompt"]}]
seed = zlib.crc32(rec["id"].encode()) % 1000003 * 7 + rec.get("sample", 0) + 7000
svgpy = task["cat"] == "svg"
for rnd in range(args.rounds):
if bad == 0:
break
how = " (the Python program, then the SVG it printed in Chrome)" if svgpy else " in Chrome (window 1024x768)"
msgs = msgs + [{"role": "assistant", "content": cur or "(no answer)"},
{"role": "user", "content": FEEDBACK.format(how=how, lines="\n".join("- " + x for x in lines),
what="program" if svgpy else "file",
lang="python" if svgpy else "html")}]
cur, n = generate(msgs, seed + rnd)
used += n
final, bad, lines = check(cur, task, os.path.join(SHOTS, f"{name}_r{rnd + 1}.png"))
if bad < best[0]:
best = (bad, final, rnd + 1)
print(f"{rec['id']} s{rec.get('sample', 0)}: badness {best[0]} (kept round {best[2]})", flush=True)
return {**rec, "completion": "(kit)" + best[1], "kit_bad": best[0], "kit_round": best[2],
"kit_first": "(kit)" + first[1], "kit_first_bad": first[0], "n_tokens": rec["n_tokens"] + used}
def safe(rec):
try:
return process(rec)
except Exception:
import traceback
print(f"{rec['id']} s{rec.get('sample', 0)}: ERROR " + traceback.format_exc(), flush=True)
return {**rec, "kit_bad": 99}
recs = [json.loads(l) for l in open(args.src, encoding="utf-8")]
t0 = time.time()
with ThreadPoolExecutor(args.workers) as ex:
out = list(ex.map(safe, recs))
os.makedirs(os.path.dirname(args.out), exist_ok=True)
with open(args.out, "w", encoding="utf-8") as f:
for r in out:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print(f"{len(out)} records, {sum(r.get('kit_bad', 1) == 0 for r in out)} render cleanly, "
f"{sum(r.get('kit_round', 0) > 0 for r in out)} repaired, {(time.time() - t0) / 60:.1f} min")