"""Toolkit module, last step: make every answer a self-contained file and repair what fails. For each record (answers to the toolkit prompts, after the read-out module): the toolkit is inlined or the svgkit program is run (scripts/kit_inline.py), the result is rendered in headless Chrome, and problems (a failing program, invalid SVG, JavaScript errors, a blank page, a static animation, overflow) go back to the model in a new turn of the same conversation (up to --rounds); a repair is kept only if it has fewer problems. The output completions hold the final self-contained ```svg / ```html answer, so visual_eval.py scores them like any other run. usage: kit_repair.py --src results/vis_kit_dm/vis.jsonl --tasks data/visual/tasks_kit.jsonl --out results/vis_kitr/vis.jsonl """ import argparse import json import os import sys import threading import time import urllib.request import zlib from concurrent.futures import ThreadPoolExecutor from transformers import AutoTokenizer sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from kit_inline import finalize # noqa: E402 from render import Renderer # noqa: E402 from visual_loop import problems # noqa: E402 THINK_END, EOS = 124902, 124900 ap = argparse.ArgumentParser() ap.add_argument("--src", required=True) ap.add_argument("--tasks", required=True, help="the toolkit prompts the answers were generated from") ap.add_argument("--out", required=True) ap.add_argument("--rounds", type=int, default=2) ap.add_argument("--think", type=int, default=6144) ap.add_argument("--reserve", type=int, default=4096) ap.add_argument("--workers", type=int, default=8) ap.add_argument("--port", type=int, default=8080) ap.add_argument("--hints", action="store_true") ap.add_argument("--tokenizer", default="models/LFM3-1.7B") args = ap.parse_args() tok = AutoTokenizer.from_pretrained(args.tokenizer) ENDS = {i for i in range(len(tok)) if (t := tok.convert_ids_to_tokens(i)) is not None and (t.endswith(".") or t.endswith("Ċ"))} SAMPLING = {"temperature": 0.6, "top_p": 0.95, "top_k": 50, "min_p": 0.0, "repeat_penalty": 1.0, "samplers": ["top_k", "top_p", "temperature"]} tasks = {json.loads(l)["id"]: json.loads(l) for l in open(args.tasks, encoding="utf-8")} SHOTS = os.path.join(os.path.dirname(args.out), "shots") os.makedirs(SHOTS, exist_ok=True) def post(body): for attempt in range(20): try: p = json.loads(urllib.request.urlopen(f"http://localhost:{args.port}/props", timeout=10).read()) body = {**body, "n_predict": min(body["n_predict"], p["default_generation_settings"]["n_ctx"] - len(body["prompt"]) - 16)} req = urllib.request.Request(f"http://localhost:{args.port}/completion", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) return json.loads(urllib.request.urlopen(req, timeout=3600).read()) except Exception: if attempt == 19: raise time.sleep(10) def generate(msgs, seed): ids = tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_dict=False) r = post({"prompt": ids, "n_predict": args.think + args.reserve, "seed": seed, "cache_prompt": False, "return_tokens": True, **SAMPLING}) gen = [t for t in r["tokens"] if t != EOS] if THINK_END in gen: return r["content"].split("", 1)[1], r["tokens_predicted"] lim = min(args.think, len(gen)) cut = next((j for j in range(lim, 0, -1) if gen[j - 1] in ENDS), lim) r2 = post({"prompt": ids + gen[:cut] + [THINK_END], "n_predict": args.reserve, "seed": seed + 17, "cache_prompt": False, **SAMPLING}) return r2["content"], cut + r2["tokens_predicted"] local = threading.local() def renderer(): if not hasattr(local, "r"): local.r = Renderer() return local.r def check(text, task, png): """(final self-contained answer, badness, feedback lines).""" final, problem, src = finalize(text, task["cat"]) kind = "svg" if task["cat"] == "svg" else "html" if problem: return final, 10, [problem] rep = renderer().render(kind, src, png) if src else None bad, lines = problems(rep, task["animated"], src, kind, args.hints) return final, bad, lines FEEDBACK = ("I ran your answer{how} and found these problems:\n{lines}\n\nFix them and give the complete corrected " "{what} in a single ```{lang} code block.") def process(rec): task = tasks[rec["id"]] name = f"{rec['id'].split('/')[-1]}_{rec.get('sample', 0)}" raw = rec["completion"].split("", 1)[1] if "" in rec["completion"] else "" final, bad, lines = check(raw, task, os.path.join(SHOTS, f"{name}_r0.png")) first, best, used, cur = (bad, final), (bad, final, 0), 0, raw msgs = [{"role": "user", "content": task["prompt"]}] seed = zlib.crc32(rec["id"].encode()) % 1000003 * 7 + rec.get("sample", 0) + 7000 svgpy = task["cat"] == "svg" for rnd in range(args.rounds): if bad == 0: break how = " (the Python program, then the SVG it printed in Chrome)" if svgpy else " in Chrome (window 1024x768)" msgs = msgs + [{"role": "assistant", "content": cur or "(no answer)"}, {"role": "user", "content": FEEDBACK.format(how=how, lines="\n".join("- " + x for x in lines), what="program" if svgpy else "file", lang="python" if svgpy else "html")}] cur, n = generate(msgs, seed + rnd) used += n final, bad, lines = check(cur, task, os.path.join(SHOTS, f"{name}_r{rnd + 1}.png")) if bad < best[0]: best = (bad, final, rnd + 1) print(f"{rec['id']} s{rec.get('sample', 0)}: badness {best[0]} (kept round {best[2]})", flush=True) return {**rec, "completion": "(kit)" + best[1], "kit_bad": best[0], "kit_round": best[2], "kit_first": "(kit)" + first[1], "kit_first_bad": first[0], "n_tokens": rec["n_tokens"] + used} def safe(rec): try: return process(rec) except Exception: import traceback print(f"{rec['id']} s{rec.get('sample', 0)}: ERROR " + traceback.format_exc(), flush=True) return {**rec, "kit_bad": 99} recs = [json.loads(l) for l in open(args.src, encoding="utf-8")] t0 = time.time() with ThreadPoolExecutor(args.workers) as ex: out = list(ex.map(safe, recs)) os.makedirs(os.path.dirname(args.out), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: for r in out: f.write(json.dumps(r, ensure_ascii=False) + "\n") print(f"{len(out)} records, {sum(r.get('kit_bad', 1) == 0 for r in out)} render cleanly, " f"{sum(r.get('kit_round', 0) > 0 for r in out)} repaired, {(time.time() - t0) / 60:.1f} min")