import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must come before torch) import json # noqa: E402 import queue # noqa: E402 import re # noqa: E402 import threading # noqa: E402 import time # noqa: E402 import gradio as gr # noqa: E402 from transformers import PreTrainedTokenizerFast # noqa: E402 from sandbox import format_result, new_workdir, run_python # noqa: E402 MODEL_ID = "RecursiveSI/flint" FAKE = os.environ.get("FLINT_FAKE") == "1" # local UI testing without the 80 GB checkpoint # Prompt + reply share this window (32K peaks around 79 GiB of the 96 GiB card with BF16 weights). MAX_CONTEXT = 32768 MAX_NEW_TOKENS = 16384 # Conservative throughput for the GPU-time estimate (measured on this Space: ~84 tok/s end to end per turn). PREFILL_TOK_S = 6000.0 DECODE_TOK_S = 80.0 GPU_OVERHEAD_S = 12 MAX_GPU_SECONDS = 240 EOS_IDS = {150001, 150003, 150005, 150018} # end_of_text, user, observation, developer tokenizer = PreTrainedTokenizerFast.from_pretrained(MODEL_ID) _tok_lock = threading.Lock() if not FAKE: import torch from flint import generate, load_flint model, _cfg = load_flint(MODEL_ID, device="cuda", dtype=torch.bfloat16, fp8_experts=False, delete_shards=True) RUN_PYTHON_TOOL = { "type": "function", "function": { "name": "run_python", "description": ( "Run a Python 3 script in a sandbox and return its exit code, stdout and stderr. Each call starts a " "fresh interpreter in a working directory that starts empty (code from the conversation is not on " "disk, so include everything the script needs); files the script writes there persist across calls in " "this conversation. " "numpy and pandas are available. There is no network access, no subprocesses and no pip install. " "Print whatever you want to see; limits: 20 s wall clock, 2 GB memory." ), "parameters": { "type": "object", "properties": {"code": {"type": "string", "description": "Complete Python source to execute."}}, "required": ["code"], }, }, } TOOLS = [RUN_PYTHON_TOOL] TOOL_TYPES = {"run_python": {"code": "string"}} AGENT = "Agent · runs Python" CHAT = "Chat" AGENT_SYSTEM = ( "You are Flint, a coding agent. You can execute Python with the run_python tool. When a task involves code, " "write it, run it to check that it works (add small tests or assertions), fix anything that fails, and only " "then give a concise final answer with the working code." ) # -------------------------------------------------------------------------------------------- # GPU: one model turn per @spaces.GPU call def gpu_seconds(prompt_tokens: int, max_new_tokens: int) -> int: est = GPU_OVERHEAD_S + prompt_tokens / PREFILL_TOK_S + int(max_new_tokens) / DECODE_TOK_S return int(min(MAX_GPU_SECONDS, est)) def _turn_duration(prompt_ids, max_new_tokens, *args, **kwargs): return gpu_seconds(len(prompt_ids), max_new_tokens) @spaces.GPU(size="xlarge", duration=_turn_duration) def flint_turn(prompt_ids: list, max_new_tokens: int, temperature: float, top_p: float, top_k: int): """Generate one assistant turn. Yields ("tok", [ids]) batches, then ("end", stats) or ("error", msg).""" t0 = time.time() budget = gpu_seconds(len(prompt_ids), max_new_tokens) - 4 q = queue.SimpleQueue() state = {"n": 0, "error": None, "timed_out": False} def should_stop(): if time.time() - t0 > budget: state["timed_out"] = True return True return False def worker(): try: state["n"] = generate( model, torch.tensor(prompt_ids, dtype=torch.long), max_new_tokens=int(max_new_tokens), eos_ids=EOS_IDS, temperature=float(temperature), top_p=float(top_p), top_k=int(top_k), on_token=q.put, should_stop=should_stop, ) except Exception as e: # reported to the UI below state["error"] = f"{type(e).__name__}: {e}" finally: q.put(None) th = threading.Thread(target=worker, daemon=True) th.start() done = False while not done: batch = [] deadline = time.time() + 0.06 while time.time() < deadline: try: item = q.get(timeout=max(0.0, deadline - time.time())) except queue.Empty: break if item is None: done = True break batch.append(item) if batch: yield ("tok", batch) th.join() if state["error"]: yield ("error", state["error"]) else: yield ("end", {"n": state["n"], "timed_out": state["timed_out"], "gpu_s": round(time.time() - t0, 1)}) def _fake_turn(prompt_ids, max_new_tokens, temperature, top_p, top_k): """Scripted stand-in for flint_turn so the UI and agent loop can be exercised without a GPU.""" prompt = tokenizer.decode(prompt_ids[-400:], skip_special_tokens=False) if "" in tokenizer.decode(prompt_ids[:400], skip_special_tokens=False) and "" not in prompt: text = ( "I'll write the function and test it before answering.Let me check it runs." "run_pythoncodedef add(a, b):\n return a + b\n\n" "assert add(2, 3) == 5\nprint('ok', add(40, 2))" ) elif "" in prompt: text = "The test passed.It works:\n\n```python\ndef add(a, b):\n return a + b\n```" else: text = "Hello! This is the **offline test model** — no weights loaded." ids = tokenizer.encode(text, add_special_tokens=False) for i in range(0, len(ids), 3): time.sleep(0.02) yield ("tok", ids[i : i + 3]) yield ("end", {"n": len(ids), "timed_out": False, "gpu_s": 0.0}) TURN = _fake_turn if FAKE else flint_turn SKIPPED = {"skipped": True} # -------------------------------------------------------------------------------------------- # Prompt building and parsing Flint's output format def encode(system: str, msgs: list, use_tools: bool, max_new_tokens: int): """Chat template -> token ids, dropping the oldest exchanges if the prompt would not fit.""" limit = MAX_CONTEXT - int(max_new_tokens) msgs = list(msgs) while True: convo = ([{"role": "system", "content": system}] if system else []) + msgs with _tok_lock: ids = tokenizer.apply_chat_template( convo, tools=TOOLS if use_tools else None, add_generation_prompt=True, tokenize=True, return_dict=True )["input_ids"] ids = list(ids) users = [i for i, m in enumerate(msgs) if m["role"] == "user"] if len(ids) <= limit or len(users) < 2: break msgs = msgs[users[1] :] # drop the oldest exchange if len(ids) > limit: raise gr.Error( f"The conversation no longer fits this demo's {MAX_CONTEXT:,}-token window " f"(prompt + reply). Clear the chat or lower Max new tokens." ) return ids _ARG_RE = re.compile(r"(.*?)\s*(.*?)(|$)", re.S) def _arg_value(tool: str, key: str, raw: str): if TOOL_TYPES.get(tool, {}).get(key) == "string": return raw try: return json.loads(raw) except (ValueError, TypeError): return raw def parse_turn(text: str): """Split a (possibly partial) turn into thinking, visible content and tool calls. Flint decides per turn whether to reason; when it does, the turn opens with .... Tool calls look like namekv. """ s = text.lstrip() thinking, rest, think_open = "", s, False if s.startswith(""): if "" in s: thinking, rest = s[len("") :].split("", 1) else: thinking, rest, think_open = s[len("") :], "", True elif s and "".startswith(s): # can't tell yet rest = "" cut = rest.find("") content = rest if cut < 0 else rest[:cut] calls = [] if cut >= 0: for m in re.finditer(r"(.*?)(|$)", rest[cut:], re.S): body, closed = m.group(1), bool(m.group(2)) # Unconstrained sampling occasionally repeats the opening tag; keep only the bare name. name = re.sub(r"<[^>]*>?", "", body.split("", 1)[0]).strip() args = {k.strip(): _arg_value(name, k.strip(), v) for k, v, _ in _ARG_RE.findall(body)} calls.append({"name": name, "arguments": args, "closed": closed}) # A partial "")): if content.endswith(""[:i]): content = content[:-i] break return thinking.strip(), content.strip(), calls, think_open # -------------------------------------------------------------------------------------------- # Rendering def _fence(code: str, lang: str = "") -> str: ticks = "```" while ticks in code: ticks += "`" return f"{ticks}{lang}\n{code.rstrip()}\n{ticks}" def _thinking_msg(thinking: str, open_: bool, secs: float): title = "💭 Thinking…" if open_ else f"💭 Thought for {secs:.1f} s" return gr.ChatMessage( role="assistant", content=thinking, metadata={"title": title, "status": "pending" if open_ else "done"} ) def _tool_msg(call: dict, result: dict | None, phase: str = "writing…"): name = call["name"] or "tool" code = call["arguments"].get("code", "") if isinstance(call["arguments"], dict) else "" body = _fence(code, "python") if code else _fence(json.dumps(call["arguments"], indent=2), "json") if result is None: title = f"🐍 {name} · {phase}" status = "pending" elif result.get("skipped"): return gr.ChatMessage(role="assistant", content=body, metadata={"title": f"🐍 {name} · not run (tool-round limit)", "status": "done"}) else: ok = result.get("exit_code") == 0 title = f"🐍 {name} · {'✓' if ok else '✗ exit ' + str(result.get('exit_code'))} · {result.get('seconds', 0)} s" status = "done" out = result.get("display", "") body += "\n\n**Output**\n\n" + _fence(out or "(no output)") return gr.ChatMessage(role="assistant", content=body, metadata={"title": title, "status": status}) def _render_turn(text: str, think_secs: float, results: list | None = None, running: int = -1): thinking, content, calls, think_open = parse_turn(text) out = [] if thinking or think_open: out.append(_thinking_msg(thinking, think_open, think_secs)) if content: out.append(gr.ChatMessage(role="assistant", content=content)) for i, c in enumerate(calls): res = results[i] if results is not None and i < len(results) else None phase = "writing…" if running < 0 else ("running…" if i == running else "queued") out.append(_tool_msg(c, res, phase)) return out # -------------------------------------------------------------------------------------------- # Agent loop (runs in the web process; each model turn is one GPU call) def run_episode(message, msgs, workdir, mode, system_prompt, max_new_tokens, max_rounds, temperature, top_p, top_k): """Yields (display_messages_for_this_episode, msgs, workdir, stats_line) as the episode progresses.""" use_tools = mode == AGENT system = (system_prompt or "").strip() msgs = list(msgs or []) + [{"role": "user", "content": message}] if use_tools and not workdir: workdir = new_workdir() shown = [] t0 = time.time() total_tokens, gpu_s, rounds, notes = 0, 0.0, 0, [] max_rounds = int(max_rounds) # Rounds 0..max_rounds-1 may run tools. A call past the budget is answered with a "budget used up" tool # response and Flint gets one final turn to answer; if it still calls a tool, the episode stops. for rnd in range(max_rounds + 2): rounds = rnd + 1 ids = encode(system, msgs, use_tools, max_new_tokens) gen, text, think_start, think_secs = [], "", None, 0.0 stats = {} for kind, payload in TURN(ids, int(max_new_tokens), float(temperature), float(top_p), int(top_k)): if kind == "tok": gen.extend(payload) with _tok_lock: text = tokenizer.decode(gen, skip_special_tokens=False) if text.endswith("�"): continue _, _, _, think_open = parse_turn(text) if think_open: think_start = think_start or time.time() think_secs = time.time() - think_start yield shown + _render_turn(text, think_secs), msgs, workdir, "Generating…" elif kind == "error": raise gr.Error(f"Generation failed: {payload}") else: stats = payload total_tokens += stats.get("n", len(gen)) gpu_s += stats.get("gpu_s", 0.0) if stats.get("timed_out"): notes.append("stopped at the GPU time limit") elif len(gen) >= int(max_new_tokens): notes.append("hit Max new tokens" + (" while still reasoning — raise it in Settings" if parse_turn(text)[3] else "")) thinking, content, calls, _ = parse_turn(text) calls = [c for c in calls if c["name"]] assistant = {"role": "assistant", "content": content} if thinking: assistant["reasoning_content"] = thinking if calls and use_tools: assistant["tool_calls"] = [ {"type": "function", "function": {"name": c["name"], "arguments": c["arguments"]}} for c in calls ] msgs.append(assistant) if not (calls and use_tools): shown += _render_turn(text, think_secs) break if rnd > max_rounds: shown += _render_turn(text, think_secs, [SKIPPED] * len(calls)) notes.append(f"stopped after {max_rounds} tool rounds") break if rnd == max_rounds: shown += _render_turn(text, think_secs, [SKIPPED] * len(calls)) for _ in calls: msgs.append({"role": "tool", "content": ( f"error: the tool budget for this request ({max_rounds} rounds) is used up, so this call was not " "run. Do not call any more tools. Give your final answer now, based on what you already know, " "and say what you could not verify.")}) yield shown, msgs, workdir, "Tool budget used up — asking for a final answer…" continue results = [] for i, c in enumerate(calls): yield shown + _render_turn(text, think_secs, results, running=i), msgs, workdir, "Running Python…" if c["name"] == "run_python" and isinstance(c["arguments"].get("code"), str): r = run_python(c["arguments"]["code"], workdir) workdir = r["workdir"] r["display"] = "\n".join(x for x in (r["stdout"].rstrip(), r["stderr"].rstrip()) if x) response = format_result(r) else: r = {"exit_code": "n/a", "seconds": 0, "display": f"Unknown tool or missing code: {c['name']}"} response = f"error: unknown tool {c['name']!r} or missing 'code' argument. Available: run_python(code)." results.append(r) msgs.append({"role": "tool", "content": response}) shown += _render_turn(text, think_secs, results) yield shown, msgs, workdir, "Thinking about the results…" dt = time.time() - t0 line = f"{rounds} turn{'s' if rounds != 1 else ''} · {total_tokens:,} tokens · {dt:.1f} s" if gpu_s: line += f" · ~{total_tokens / max(gpu_s, 1e-6):.0f} tok/s incl. GPU setup" if notes: line += " · " + ", ".join(notes) yield shown, msgs, workdir, line def respond(message, chat, msgs, workdir, mode, system_prompt, max_new_tokens, max_rounds, temperature, top_p, top_k): message = (message or "").strip() if not message: raise gr.Error("Type a message first.") base = list(chat or []) + [gr.ChatMessage(role="user", content=message)] yield base, msgs, workdir, "Waiting for a GPU…", "" for shown, m, w, line in run_episode(message, msgs, workdir, mode, system_prompt, max_new_tokens, max_rounds, temperature, top_p, top_k): yield base + shown, m, w, line, "" def ask_flint(task: str, use_python: bool = True, max_new_tokens: int = 6144, max_rounds: int = 4) -> str: """Ask Recursive's Flint coding model to solve a task, optionally letting it run Python to check its work. Args: task: The coding question or task. use_python: Let Flint execute Python in a sandbox (no network) and iterate on the results. max_new_tokens: Token budget per model turn, reasoning included (256-16384). max_rounds: Maximum number of tool-call rounds before Flint must answer. Returns: Flint's final answer as Markdown, followed by the tool calls it made. """ mode = AGENT if use_python else CHAT system = AGENT_SYSTEM if use_python else "" final = None for final in run_episode(task, [], None, mode, system, max(256, min(MAX_NEW_TOKENS, int(max_new_tokens))), max(0, min(8, int(max_rounds))), 1.0, 0.95, 20): pass shown, msgs, _, line = final answer = next((m["content"] for m in reversed(msgs) if m["role"] == "assistant" and m.get("content")), "") tools = [m for m in shown if (m.metadata or {}).get("title", "").startswith("🐍")] parts = [answer or "(No final answer — Flint used its whole token budget reasoning. Raise max_new_tokens.)"] if not answer: reasoning = next((m.get("reasoning_content", "") for m in reversed(msgs) if m["role"] == "assistant"), "") if reasoning: tail = reasoning[-2500:] parts.append("**End of Flint's reasoning**\n\n" + ("…" if len(reasoning) > 2500 else "") + tail) if tools: parts.append("\n---\n**Tool calls**\n") parts += [f"**{t.metadata['title']}**\n\n{t.content}" for t in tools] parts.append(f"\n_{line}_") return "\n\n".join(parts) # -------------------------------------------------------------------------------------------- # UI EXAMPLES = [ "Write a function `nth_prime(n)`, check it against a brute-force version for n ≤ 300, then time nth_prime(20000).", "This binary search sometimes loops forever or misses items. Write a failing test, then fix it.\n\n" "```python\ndef search(xs, target):\n lo, hi = 0, len(xs)\n while lo < hi:\n mid = (lo + hi) // 2\n" " if xs[mid] < target:\n lo = mid\n elif xs[mid] > target:\n hi = mid - 1\n" " else:\n return mid\n return -1\n```", "Implement an LRU cache with O(1) get/put and prove the eviction order with unittest.", "Simulate 1,000,000 rolls of two dice with numpy and compare the empirical distribution of the sum with the " "exact one in a small table.", "Here's some sales data. Which city has the highest average price, and by how much?\n\n" "city,price\nCape Town,120\nDurban,95\nJohannesburg,130\nCape Town,140\nDurban,105\nJohannesburg,90\n" "Pretoria,110\nCape Town,100", ] CSS = """ #hdr {text-align:center; margin: 4px 0 2px} #hdr h1 {font-size: 1.9rem; margin: 0 0 4px} #hdr p {margin: 2px 0; opacity: .85} .chips {display:flex; gap:6px; justify-content:center; flex-wrap:wrap; margin:8px 0 2px} .chips span {font-size:.78rem; padding:2px 9px; border-radius:999px; border:1px solid var(--border-color-primary);} #stats {font-size:.82rem; opacity:.75; min-height:1.4em} footer {display:none !important} """ HEADER = """

🪨 Flint Agent

Recursive's open coding-agent model, running here with a live Python tool so you can watch it write, run and fix code.

39.8B MoE · ~2.5B active256 experts, top-6SWE-bench Verified 74.2% Terminal-Bench 2.1 54.9%Apache-2.0
""" with gr.Blocks(title="Flint Agent") as demo: gr.HTML(HEADER) msgs_state = gr.State([]) workdir_state = gr.State(None) with gr.Row(): mode = gr.Radio([AGENT, CHAT], value=AGENT, show_label=False, scale=3) clear = gr.Button("New chat", variant="secondary", scale=1, size="sm") chatbot = gr.Chatbot( label="Flint", height=600, show_label=False, placeholder="**Ask Flint a coding question.** In agent mode it can run Python (no network) to test its work.", ) with gr.Row(): box = gr.Textbox(placeholder="Ask a coding question…", show_label=False, scale=8, lines=1, max_lines=12, autofocus=True) send = gr.Button("Send", variant="primary", scale=1) stop = gr.Button("Stop", variant="stop", scale=1, visible=True) stats = gr.Markdown("", elem_id="stats") with gr.Accordion("Settings", open=False): system_prompt = gr.Textbox(label="System prompt", lines=3, value=AGENT_SYSTEM) with gr.Row(): max_new_tokens = gr.Slider(256, MAX_NEW_TOKENS, value=6144, step=256, label="Max new tokens per turn", info="Reasoning counts toward this.") max_rounds = gr.Slider(0, 8, value=4, step=1, label="Max tool rounds") with gr.Row(): temperature = gr.Slider(0.0, 1.5, value=1.0, step=0.05, label="Temperature", info="0 = greedy") top_p = gr.Slider(0.05, 1.0, value=0.95, step=0.05, label="Top-p") top_k = gr.Slider(0, 200, value=20, step=1, label="Top-k", info="0 = off") gr.Examples(EXAMPLES, inputs=box, label="Try one") gr.Markdown( "Each model turn runs on a ZeroGPU **xlarge** slice (96 GB RTX PRO 6000) with the BF16 weights, " "through a plain-PyTorch port of the reference SGLang model (vendored from " "[mrfakename/flint](https://huggingface.co/spaces/mrfakename/flint)). Python runs in a throwaway " "sandbox with no network, no subprocesses, 20 s and 2 GB limits. " "Prompt + reply share a 32K-token window. Model: [RecursiveSI/flint](https://huggingface.co/RecursiveSI/flint) " "· sampling defaults follow the model card (T 1.0, top-p 0.95, top-k 20)." ) def _mode_changed(m, current): if m == CHAT and current.strip() == AGENT_SYSTEM: return "" if m == AGENT and not current.strip(): return AGENT_SYSTEM return current mode.change(_mode_changed, [mode, system_prompt], system_prompt, api_visibility="private") inputs = [box, chatbot, msgs_state, workdir_state, mode, system_prompt, max_new_tokens, max_rounds, temperature, top_p, top_k] outputs = [chatbot, msgs_state, workdir_state, stats, box] ev1 = box.submit(respond, inputs, outputs, concurrency_limit=4, api_visibility="private") ev2 = send.click(respond, inputs, outputs, concurrency_limit=4, api_visibility="private") stop.click(None, None, None, cancels=[ev1, ev2], api_visibility="private") clear.click(lambda: ([], [], None, ""), None, [chatbot, msgs_state, workdir_state, stats], api_visibility="private") gr.api(ask_flint, api_name="ask_flint", concurrency_limit=4) if __name__ == "__main__": demo.queue(default_concurrency_limit=4).launch(theme=gr.themes.Soft(primary_hue="amber"), css=CSS, mcp_server=True)