flint-agent / app.py
joeWorld's picture
Tolerate repeated <tool_call> tags in tool names
40f8f73 verified
Raw History Blame Contribute Delete
24.3 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must come before torch)
import json # noqa: E402
import queue # noqa: E402
import re # noqa: E402
import threading # noqa: E402
import time # noqa: E402
import gradio as gr # noqa: E402
from transformers import PreTrainedTokenizerFast # noqa: E402
from sandbox import format_result, new_workdir, run_python # noqa: E402
MODEL_ID = "RecursiveSI/flint"
FAKE = os.environ.get("FLINT_FAKE") == "1" # local UI testing without the 80 GB checkpoint
# Prompt + reply share this window (32K peaks around 79 GiB of the 96 GiB card with BF16 weights).
MAX_CONTEXT = 32768
MAX_NEW_TOKENS = 16384
# Conservative throughput for the GPU-time estimate (measured on this Space: ~84 tok/s end to end per turn).
PREFILL_TOK_S = 6000.0
DECODE_TOK_S = 80.0
GPU_OVERHEAD_S = 12
MAX_GPU_SECONDS = 240
EOS_IDS = {150001, 150003, 150005, 150018} # end_of_text, user, observation, developer
tokenizer = PreTrainedTokenizerFast.from_pretrained(MODEL_ID)
_tok_lock = threading.Lock()
if not FAKE:
import torch
from flint import generate, load_flint
model, _cfg = load_flint(MODEL_ID, device="cuda", dtype=torch.bfloat16, fp8_experts=False, delete_shards=True)
RUN_PYTHON_TOOL = {
"type": "function",
"function": {
"name": "run_python",
"description": (
"Run a Python 3 script in a sandbox and return its exit code, stdout and stderr. Each call starts a "
"fresh interpreter in a working directory that starts empty (code from the conversation is not on "
"disk, so include everything the script needs); files the script writes there persist across calls in "
"this conversation. "
"numpy and pandas are available. There is no network access, no subprocesses and no pip install. "
"Print whatever you want to see; limits: 20 s wall clock, 2 GB memory."
),
"parameters": {
"type": "object",
"properties": {"code": {"type": "string", "description": "Complete Python source to execute."}},
"required": ["code"],
},
},
}
TOOLS = [RUN_PYTHON_TOOL]
TOOL_TYPES = {"run_python": {"code": "string"}}
AGENT = "Agent · runs Python"
CHAT = "Chat"
AGENT_SYSTEM = (
"You are Flint, a coding agent. You can execute Python with the run_python tool. When a task involves code, "
"write it, run it to check that it works (add small tests or assertions), fix anything that fails, and only "
"then give a concise final answer with the working code."
)
# --------------------------------------------------------------------------------------------
# GPU: one model turn per @spaces.GPU call
def gpu_seconds(prompt_tokens: int, max_new_tokens: int) -> int:
est = GPU_OVERHEAD_S + prompt_tokens / PREFILL_TOK_S + int(max_new_tokens) / DECODE_TOK_S
return int(min(MAX_GPU_SECONDS, est))
def _turn_duration(prompt_ids, max_new_tokens, *args, **kwargs):
return gpu_seconds(len(prompt_ids), max_new_tokens)
@spaces.GPU(size="xlarge", duration=_turn_duration)
def flint_turn(prompt_ids: list, max_new_tokens: int, temperature: float, top_p: float, top_k: int):
"""Generate one assistant turn. Yields ("tok", [ids]) batches, then ("end", stats) or ("error", msg)."""
t0 = time.time()
budget = gpu_seconds(len(prompt_ids), max_new_tokens) - 4
q = queue.SimpleQueue()
state = {"n": 0, "error": None, "timed_out": False}
def should_stop():
if time.time() - t0 > budget:
state["timed_out"] = True
return True
return False
def worker():
try:
state["n"] = generate(
model,
torch.tensor(prompt_ids, dtype=torch.long),
max_new_tokens=int(max_new_tokens),
eos_ids=EOS_IDS,
temperature=float(temperature),
top_p=float(top_p),
top_k=int(top_k),
on_token=q.put,
should_stop=should_stop,
)
except Exception as e: # reported to the UI below
state["error"] = f"{type(e).__name__}: {e}"
finally:
q.put(None)
th = threading.Thread(target=worker, daemon=True)
th.start()
done = False
while not done:
batch = []
deadline = time.time() + 0.06
while time.time() < deadline:
try:
item = q.get(timeout=max(0.0, deadline - time.time()))
except queue.Empty:
break
if item is None:
done = True
break
batch.append(item)
if batch:
yield ("tok", batch)
th.join()
if state["error"]:
yield ("error", state["error"])
else:
yield ("end", {"n": state["n"], "timed_out": state["timed_out"], "gpu_s": round(time.time() - t0, 1)})
def _fake_turn(prompt_ids, max_new_tokens, temperature, top_p, top_k):
"""Scripted stand-in for flint_turn so the UI and agent loop can be exercised without a GPU."""
prompt = tokenizer.decode(prompt_ids[-400:], skip_special_tokens=False)
if "<tools>" in tokenizer.decode(prompt_ids[:400], skip_special_tokens=False) and "<tool_response>" not in prompt:
text = (
"<think>I'll write the function and test it before answering.</think>Let me check it runs."
"<tool_call>run_python<arg_key>code</arg_key><arg_value>def add(a, b):\n return a + b\n\n"
"assert add(2, 3) == 5\nprint('ok', add(40, 2))</arg_value></tool_call>"
)
elif "<tool_response>" in prompt:
text = "<think>The test passed.</think>It works:\n\n```python\ndef add(a, b):\n return a + b\n```"
else:
text = "Hello! This is the **offline test model** — no weights loaded."
ids = tokenizer.encode(text, add_special_tokens=False)
for i in range(0, len(ids), 3):
time.sleep(0.02)
yield ("tok", ids[i : i + 3])
yield ("end", {"n": len(ids), "timed_out": False, "gpu_s": 0.0})
TURN = _fake_turn if FAKE else flint_turn
SKIPPED = {"skipped": True}
# --------------------------------------------------------------------------------------------
# Prompt building and parsing Flint's output format
def encode(system: str, msgs: list, use_tools: bool, max_new_tokens: int):
"""Chat template -> token ids, dropping the oldest exchanges if the prompt would not fit."""
limit = MAX_CONTEXT - int(max_new_tokens)
msgs = list(msgs)
while True:
convo = ([{"role": "system", "content": system}] if system else []) + msgs
with _tok_lock:
ids = tokenizer.apply_chat_template(
convo, tools=TOOLS if use_tools else None, add_generation_prompt=True, tokenize=True, return_dict=True
)["input_ids"]
ids = list(ids)
users = [i for i, m in enumerate(msgs) if m["role"] == "user"]
if len(ids) <= limit or len(users) < 2:
break
msgs = msgs[users[1] :] # drop the oldest exchange
if len(ids) > limit:
raise gr.Error(
f"The conversation no longer fits this demo's {MAX_CONTEXT:,}-token window "
f"(prompt + reply). Clear the chat or lower Max new tokens."
)
return ids
_ARG_RE = re.compile(r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)(</arg_value>|$)", re.S)
def _arg_value(tool: str, key: str, raw: str):
if TOOL_TYPES.get(tool, {}).get(key) == "string":
return raw
try:
return json.loads(raw)
except (ValueError, TypeError):
return raw
def parse_turn(text: str):
"""Split a (possibly partial) turn into thinking, visible content and tool calls.
Flint decides per turn whether to reason; when it does, the turn opens with <think>...</think>.
Tool calls look like <tool_call>name<arg_key>k</arg_key><arg_value>v</arg_value></tool_call>.
"""
s = text.lstrip()
thinking, rest, think_open = "", s, False
if s.startswith("<think>"):
if "</think>" in s:
thinking, rest = s[len("<think>") :].split("</think>", 1)
else:
thinking, rest, think_open = s[len("<think>") :], "", True
elif s and "<think>".startswith(s): # can't tell yet
rest = ""
cut = rest.find("<tool_call>")
content = rest if cut < 0 else rest[:cut]
calls = []
if cut >= 0:
for m in re.finditer(r"<tool_call>(.*?)(</tool_call>|$)", rest[cut:], re.S):
body, closed = m.group(1), bool(m.group(2))
# Unconstrained sampling occasionally repeats the opening tag; keep only the bare name.
name = re.sub(r"<[^>]*>?", "", body.split("<arg_key>", 1)[0]).strip()
args = {k.strip(): _arg_value(name, k.strip(), v) for k, v, _ in _ARG_RE.findall(body)}
calls.append({"name": name, "arguments": args, "closed": closed})
# A partial "<tool_c..." at the very end shouldn't flash in the content while streaming.
for i in range(1, len("<tool_call>")):
if content.endswith("<tool_call>"[:i]):
content = content[:-i]
break
return thinking.strip(), content.strip(), calls, think_open
# --------------------------------------------------------------------------------------------
# Rendering
def _fence(code: str, lang: str = "") -> str:
ticks = "```"
while ticks in code:
ticks += "`"
return f"{ticks}{lang}\n{code.rstrip()}\n{ticks}"
def _thinking_msg(thinking: str, open_: bool, secs: float):
title = "💭 Thinking…" if open_ else f"💭 Thought for {secs:.1f} s"
return gr.ChatMessage(
role="assistant", content=thinking, metadata={"title": title, "status": "pending" if open_ else "done"}
)
def _tool_msg(call: dict, result: dict | None, phase: str = "writing…"):
name = call["name"] or "tool"
code = call["arguments"].get("code", "") if isinstance(call["arguments"], dict) else ""
body = _fence(code, "python") if code else _fence(json.dumps(call["arguments"], indent=2), "json")
if result is None:
title = f"🐍 {name} · {phase}"
status = "pending"
elif result.get("skipped"):
return gr.ChatMessage(role="assistant", content=body,
metadata={"title": f"🐍 {name} · not run (tool-round limit)", "status": "done"})
else:
ok = result.get("exit_code") == 0
title = f"🐍 {name} · {'✓' if ok else '✗ exit ' + str(result.get('exit_code'))} · {result.get('seconds', 0)} s"
status = "done"
out = result.get("display", "")
body += "\n\n**Output**\n\n" + _fence(out or "(no output)")
return gr.ChatMessage(role="assistant", content=body, metadata={"title": title, "status": status})
def _render_turn(text: str, think_secs: float, results: list | None = None, running: int = -1):
thinking, content, calls, think_open = parse_turn(text)
out = []
if thinking or think_open:
out.append(_thinking_msg(thinking, think_open, think_secs))
if content:
out.append(gr.ChatMessage(role="assistant", content=content))
for i, c in enumerate(calls):
res = results[i] if results is not None and i < len(results) else None
phase = "writing…" if running < 0 else ("running…" if i == running else "queued")
out.append(_tool_msg(c, res, phase))
return out
# --------------------------------------------------------------------------------------------
# Agent loop (runs in the web process; each model turn is one GPU call)
def run_episode(message, msgs, workdir, mode, system_prompt, max_new_tokens, max_rounds, temperature, top_p, top_k):
"""Yields (display_messages_for_this_episode, msgs, workdir, stats_line) as the episode progresses."""
use_tools = mode == AGENT
system = (system_prompt or "").strip()
msgs = list(msgs or []) + [{"role": "user", "content": message}]
if use_tools and not workdir:
workdir = new_workdir()
shown = []
t0 = time.time()
total_tokens, gpu_s, rounds, notes = 0, 0.0, 0, []
max_rounds = int(max_rounds)
# Rounds 0..max_rounds-1 may run tools. A call past the budget is answered with a "budget used up" tool
# response and Flint gets one final turn to answer; if it still calls a tool, the episode stops.
for rnd in range(max_rounds + 2):
rounds = rnd + 1
ids = encode(system, msgs, use_tools, max_new_tokens)
gen, text, think_start, think_secs = [], "", None, 0.0
stats = {}
for kind, payload in TURN(ids, int(max_new_tokens), float(temperature), float(top_p), int(top_k)):
if kind == "tok":
gen.extend(payload)
with _tok_lock:
text = tokenizer.decode(gen, skip_special_tokens=False)
if text.endswith("�"):
continue
_, _, _, think_open = parse_turn(text)
if think_open:
think_start = think_start or time.time()
think_secs = time.time() - think_start
yield shown + _render_turn(text, think_secs), msgs, workdir, "Generating…"
elif kind == "error":
raise gr.Error(f"Generation failed: {payload}")
else:
stats = payload
total_tokens += stats.get("n", len(gen))
gpu_s += stats.get("gpu_s", 0.0)
if stats.get("timed_out"):
notes.append("stopped at the GPU time limit")
elif len(gen) >= int(max_new_tokens):
notes.append("hit Max new tokens" + (" while still reasoning — raise it in Settings"
if parse_turn(text)[3] else ""))
thinking, content, calls, _ = parse_turn(text)
calls = [c for c in calls if c["name"]]
assistant = {"role": "assistant", "content": content}
if thinking:
assistant["reasoning_content"] = thinking
if calls and use_tools:
assistant["tool_calls"] = [
{"type": "function", "function": {"name": c["name"], "arguments": c["arguments"]}} for c in calls
]
msgs.append(assistant)
if not (calls and use_tools):
shown += _render_turn(text, think_secs)
break
if rnd > max_rounds:
shown += _render_turn(text, think_secs, [SKIPPED] * len(calls))
notes.append(f"stopped after {max_rounds} tool rounds")
break
if rnd == max_rounds:
shown += _render_turn(text, think_secs, [SKIPPED] * len(calls))
for _ in calls:
msgs.append({"role": "tool", "content": (
f"error: the tool budget for this request ({max_rounds} rounds) is used up, so this call was not "
"run. Do not call any more tools. Give your final answer now, based on what you already know, "
"and say what you could not verify.")})
yield shown, msgs, workdir, "Tool budget used up — asking for a final answer…"
continue
results = []
for i, c in enumerate(calls):
yield shown + _render_turn(text, think_secs, results, running=i), msgs, workdir, "Running Python…"
if c["name"] == "run_python" and isinstance(c["arguments"].get("code"), str):
r = run_python(c["arguments"]["code"], workdir)
workdir = r["workdir"]
r["display"] = "\n".join(x for x in (r["stdout"].rstrip(), r["stderr"].rstrip()) if x)
response = format_result(r)
else:
r = {"exit_code": "n/a", "seconds": 0, "display": f"Unknown tool or missing code: {c['name']}"}
response = f"error: unknown tool {c['name']!r} or missing 'code' argument. Available: run_python(code)."
results.append(r)
msgs.append({"role": "tool", "content": response})
shown += _render_turn(text, think_secs, results)
yield shown, msgs, workdir, "Thinking about the results…"
dt = time.time() - t0
line = f"{rounds} turn{'s' if rounds != 1 else ''} · {total_tokens:,} tokens · {dt:.1f} s"
if gpu_s:
line += f" · ~{total_tokens / max(gpu_s, 1e-6):.0f} tok/s incl. GPU setup"
if notes:
line += " · " + ", ".join(notes)
yield shown, msgs, workdir, line
def respond(message, chat, msgs, workdir, mode, system_prompt, max_new_tokens, max_rounds, temperature, top_p,
top_k):
message = (message or "").strip()
if not message:
raise gr.Error("Type a message first.")
base = list(chat or []) + [gr.ChatMessage(role="user", content=message)]
yield base, msgs, workdir, "Waiting for a GPU…", ""
for shown, m, w, line in run_episode(message, msgs, workdir, mode, system_prompt, max_new_tokens, max_rounds,
temperature, top_p, top_k):
yield base + shown, m, w, line, ""
def ask_flint(task: str, use_python: bool = True, max_new_tokens: int = 6144, max_rounds: int = 4) -> str:
"""Ask Recursive's Flint coding model to solve a task, optionally letting it run Python to check its work.
Args:
task: The coding question or task.
use_python: Let Flint execute Python in a sandbox (no network) and iterate on the results.
max_new_tokens: Token budget per model turn, reasoning included (256-16384).
max_rounds: Maximum number of tool-call rounds before Flint must answer.
Returns:
Flint's final answer as Markdown, followed by the tool calls it made.
"""
mode = AGENT if use_python else CHAT
system = AGENT_SYSTEM if use_python else ""
final = None
for final in run_episode(task, [], None, mode, system, max(256, min(MAX_NEW_TOKENS, int(max_new_tokens))),
max(0, min(8, int(max_rounds))), 1.0, 0.95, 20):
pass
shown, msgs, _, line = final
answer = next((m["content"] for m in reversed(msgs) if m["role"] == "assistant" and m.get("content")), "")
tools = [m for m in shown if (m.metadata or {}).get("title", "").startswith("🐍")]
parts = [answer or "(No final answer — Flint used its whole token budget reasoning. Raise max_new_tokens.)"]
if not answer:
reasoning = next((m.get("reasoning_content", "") for m in reversed(msgs) if m["role"] == "assistant"), "")
if reasoning:
tail = reasoning[-2500:]
parts.append("**End of Flint's reasoning**\n\n" + ("…" if len(reasoning) > 2500 else "") + tail)
if tools:
parts.append("\n---\n**Tool calls**\n")
parts += [f"**{t.metadata['title']}**\n\n{t.content}" for t in tools]
parts.append(f"\n_{line}_")
return "\n\n".join(parts)
# --------------------------------------------------------------------------------------------
# UI
EXAMPLES = [
"Write a function `nth_prime(n)`, check it against a brute-force version for n ≤ 300, then time nth_prime(20000).",
"This binary search sometimes loops forever or misses items. Write a failing test, then fix it.\n\n"
"```python\ndef search(xs, target):\n lo, hi = 0, len(xs)\n while lo < hi:\n mid = (lo + hi) // 2\n"
" if xs[mid] < target:\n lo = mid\n elif xs[mid] > target:\n hi = mid - 1\n"
" else:\n return mid\n return -1\n```",
"Implement an LRU cache with O(1) get/put and prove the eviction order with unittest.",
"Simulate 1,000,000 rolls of two dice with numpy and compare the empirical distribution of the sum with the "
"exact one in a small table.",
"Here's some sales data. Which city has the highest average price, and by how much?\n\n"
"city,price\nCape Town,120\nDurban,95\nJohannesburg,130\nCape Town,140\nDurban,105\nJohannesburg,90\n"
"Pretoria,110\nCape Town,100",
]
CSS = """
#hdr {text-align:center; margin: 4px 0 2px}
#hdr h1 {font-size: 1.9rem; margin: 0 0 4px}
#hdr p {margin: 2px 0; opacity: .85}
.chips {display:flex; gap:6px; justify-content:center; flex-wrap:wrap; margin:8px 0 2px}
.chips span {font-size:.78rem; padding:2px 9px; border-radius:999px; border:1px solid var(--border-color-primary);}
#stats {font-size:.82rem; opacity:.75; min-height:1.4em}
footer {display:none !important}
"""
HEADER = """
<div id="hdr">
<h1>🪨 Flint Agent</h1>
<p>Recursive's open coding-agent model, running here with a live Python tool so you can watch it write, run and fix code.</p>
<div class="chips"><span>39.8B MoE · ~2.5B active</span><span>256 experts, top-6</span><span>SWE-bench Verified 74.2%</span>
<span>Terminal-Bench 2.1 54.9%</span><span>Apache-2.0</span></div>
</div>
"""
with gr.Blocks(title="Flint Agent") as demo:
gr.HTML(HEADER)
msgs_state = gr.State([])
workdir_state = gr.State(None)
with gr.Row():
mode = gr.Radio([AGENT, CHAT], value=AGENT, show_label=False, scale=3)
clear = gr.Button("New chat", variant="secondary", scale=1, size="sm")
chatbot = gr.Chatbot(
label="Flint",
height=600,
show_label=False,
placeholder="**Ask Flint a coding question.** In agent mode it can run Python (no network) to test its work.",
)
with gr.Row():
box = gr.Textbox(placeholder="Ask a coding question…", show_label=False, scale=8, lines=1, max_lines=12,
autofocus=True)
send = gr.Button("Send", variant="primary", scale=1)
stop = gr.Button("Stop", variant="stop", scale=1, visible=True)
stats = gr.Markdown("", elem_id="stats")
with gr.Accordion("Settings", open=False):
system_prompt = gr.Textbox(label="System prompt", lines=3, value=AGENT_SYSTEM)
with gr.Row():
max_new_tokens = gr.Slider(256, MAX_NEW_TOKENS, value=6144, step=256, label="Max new tokens per turn",
info="Reasoning counts toward this.")
max_rounds = gr.Slider(0, 8, value=4, step=1, label="Max tool rounds")
with gr.Row():
temperature = gr.Slider(0.0, 1.5, value=1.0, step=0.05, label="Temperature", info="0 = greedy")
top_p = gr.Slider(0.05, 1.0, value=0.95, step=0.05, label="Top-p")
top_k = gr.Slider(0, 200, value=20, step=1, label="Top-k", info="0 = off")
gr.Examples(EXAMPLES, inputs=box, label="Try one")
gr.Markdown(
"Each model turn runs on a ZeroGPU **xlarge** slice (96 GB RTX PRO 6000) with the BF16 weights, "
"through a plain-PyTorch port of the reference SGLang model (vendored from "
"[mrfakename/flint](https://huggingface.co/spaces/mrfakename/flint)). Python runs in a throwaway "
"sandbox with no network, no subprocesses, 20 s and 2 GB limits. "
"Prompt + reply share a 32K-token window. Model: [RecursiveSI/flint](https://huggingface.co/RecursiveSI/flint) "
"· sampling defaults follow the model card (T 1.0, top-p 0.95, top-k 20)."
)
def _mode_changed(m, current):
if m == CHAT and current.strip() == AGENT_SYSTEM:
return ""
if m == AGENT and not current.strip():
return AGENT_SYSTEM
return current
mode.change(_mode_changed, [mode, system_prompt], system_prompt, api_visibility="private")
inputs = [box, chatbot, msgs_state, workdir_state, mode, system_prompt, max_new_tokens, max_rounds, temperature,
top_p, top_k]
outputs = [chatbot, msgs_state, workdir_state, stats, box]
ev1 = box.submit(respond, inputs, outputs, concurrency_limit=4, api_visibility="private")
ev2 = send.click(respond, inputs, outputs, concurrency_limit=4, api_visibility="private")
stop.click(None, None, None, cancels=[ev1, ev2], api_visibility="private")
clear.click(lambda: ([], [], None, ""), None, [chatbot, msgs_state, workdir_state, stats], api_visibility="private")
gr.api(ask_flint, api_name="ask_flint", concurrency_limit=4)
if __name__ == "__main__":
demo.queue(default_concurrency_limit=4).launch(theme=gr.themes.Soft(primary_hue="amber"), css=CSS,
mcp_server=True)