"""Live side-by-side: base Liquid models and their tuned adapters on the same seeded MiniWoB task. Each request runs the base model and the tuned model on one task seed you choose, with the repository's own prompts, parsers and episode loops, then replays both episodes into frames. These are fresh samples on ZeroGPU hardware, not the locked evaluations: the published numbers come from the recorded runs linked below. """ import subprocess import sys import tempfile from contextlib import closing from pathlib import Path HERE = Path(__file__).parent sys.path.insert(0, str(HERE)) subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"], check=True) import gradio as gr # noqa: E402 import spaces # noqa: E402 import torch # noqa: E402 from peft import PeftModel # noqa: E402 from PIL import Image # noqa: E402 from transformers import ( # noqa: E402 AutoModelForCausalLM, AutoModelForImageTextToText, AutoProcessor, AutoTokenizer, GenerationConfig, ) from browser_control.book_flight import BOOK_FLIGHT_TOOLS # noqa: E402 from browser_control.book_flight_evaluation import collect_book_flight_episode # noqa: E402 from browser_control.evaluation import evaluate_click_test # noqa: E402 from browser_control.lfm3 import LFM3Policy # noqa: E402 from browser_control.lfm3_vision import LFM3VisionPolicy # noqa: E402 from browser_control.local_browsergym import LocalBrowserGymEnvironment # noqa: E402 from browser_control.miniwob_assets import MiniwobAssetServer, ensure_miniwob_assets # noqa: E402 from browser_control.replay import side_by_side_frames # noqa: E402 from browser_control.transformers_policy import TRAINING_DECODER, DecoderConfig, TransformersPolicy # noqa: E402 from browser_control.vision_browser import VisionBrowserGymEnvironment, _ScreenshotCapture # noqa: E402 from browser_control.vision_evaluation import collect_vision_sequence # noqa: E402 from replay_tools import replay, replay_vision, single_step # noqa: E402 from browser_session import browser_session # noqa: E402 # Copies of the published adapters (same bytes as the model repos), so the Space needs no token. ADAPTERS = HERE / "adapters" LFM3_REVISION = "ac2586a34f1a1c03329dbdcdefabd8b6478ddaa7" VL_REVISION = "ba44bd51db20a2753f28c273a1571ccaa210d954" SAMPLED = DecoderConfig("sampling", True, 128, temperature=1.0, top_p=0.95) GREEDY = DecoderConfig("greedy", False, 128) EXPLICIT = dict(top_k=0, repetition_penalty=1.0) def plain_generation_config(model, tokenizer): """Don't inherit checkpoint sampling defaults; the decoder is set per call, as in evaluation.""" model.generation_config = GenerationConfig( bos_token_id=tokenizer.bos_token_id, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.pad_token_id, **EXPLICIT) class Device: index = 0 # LFM2-350M with its GRPO adapter (seed 23). lfm2_tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-350M") lfm2_base = AutoModelForCausalLM.from_pretrained("LiquidAI/LFM2-350M", dtype=torch.float16).eval() lfm2 = PeftModel.from_pretrained(lfm2_base, str(ADAPTERS / "lfm2-click-test"), torch_device="cpu").to("cuda").eval() # LFM3-1.7B with its book-flight SFT adapter. lfm3_tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM3-1.7B", revision=LFM3_REVISION, trust_remote_code=True, padding_side="left") lfm3_base = AutoModelForCausalLM.from_pretrained( "LiquidAI/LFM3-1.7B", revision=LFM3_REVISION, trust_remote_code=True, dtype=torch.float16, attn_implementation="sdpa" ).eval() plain_generation_config(lfm3_base, lfm3_tokenizer) lfm3 = PeftModel.from_pretrained(lfm3_base, str(ADAPTERS / "lfm3-book-flight"), torch_device="cpu").to("cuda").eval() # LFM3-VL-2B with the click-sequence SFT adapter (and the click-color one it started from). vl_processor = AutoProcessor.from_pretrained("LiquidAI/LFM3-VL-2B", revision=VL_REVISION, trust_remote_code=True) vl_base = AutoModelForImageTextToText.from_pretrained( "LiquidAI/LFM3-VL-2B", revision=VL_REVISION, trust_remote_code=True, dtype=torch.float16, attn_implementation="sdpa" ) vl = PeftModel.from_pretrained(vl_base, str(ADAPTERS / "lfm3-vl-sequence"), adapter_name="sequence-sft", torch_device="cpu") vl.load_adapter(str(ADAPTERS / "lfm3-vl-grpo"), adapter_name="sft-then-grpo", torch_device="cpu") vl = vl.to("cuda").eval() MINIWOB_ROOT = ensure_miniwob_assets(HERE / "miniwob-plusplus") SERVER = MiniwobAssetServer(MINIWOB_ROOT) SERVER.__enter__() def gif(frames, title): path = Path(tempfile.mkdtemp()) / "side-by-side.gif" palette = [f.quantize(colors=256, method=Image.Quantize.MEDIANCUT, dither=Image.Dither.NONE) for f in frames] palette[0].save(path, save_all=True, append_images=palette[1:], duration=[1400] * (len(frames) - 1) + [3200], loop=0, optimize=True) return str(path) def frames_of(directory): return [Image.open(p).convert("RGB") for p in sorted(Path(directory).glob("frame-*.png"))] @browser_session() def run_text_pair(task, seed, episodes, labels): """Replay each recorded episode in the seeded task and compose them side by side.""" columns, outcomes, notes = [], [], [] with closing(_ScreenshotCapture.miniwob(task, miniwob_url=SERVER.base_url)) as env: for episode, label in zip(episodes, labels): out = Path(tempfile.mkdtemp()) summary = replay(env, episode, out, 1, 3) columns.append(frames_of(out)) outcomes.append(bool(episode.get("success"))) last = (episode["steps"][-1].get("text") or "") if episode["steps"] else "" notes.append(f"**{label}:** {'success' if episode.get('success') else 'failure'}, " f"{len(episode['steps'])} step(s), ended by {episode.get('stop_reason', 'done')}; " f"replay {'matches' if summary['matches'] else 'DIFFERS'}\n\nLast output: `{' '.join(last.split())[:300]}`") frames = side_by_side_frames(columns, labels, outcomes, f"{task}, seed {seed}, live samples") return gif(frames, task), "\n\n".join(notes) # GPU time is spent only on model inference; episodes are replayed and drawn afterwards on the CPU. @spaces.GPU(duration=20) @browser_session() def click_test_episodes(seed): episodes = [] for adapter_on in (False, True): policy = TransformersPolicy(model=lfm2, tokenizer=lfm2_tokenizer, torch=torch, device=Device(), decoder=TRAINING_DECODER) factory = lambda: LocalBrowserGymEnvironment.miniwob("click-test", miniwob_url=SERVER.base_url) # noqa: E731 if adapter_on: report = evaluate_click_test(factory, policy, [seed]) else: with lfm2.disable_adapter(): report = evaluate_click_test(factory, policy, [seed]) episodes.append(single_step(report.to_dict()["episodes"][0])) return episodes def click_test(seed): seed = int(seed) return run_text_pair("click-test", seed, click_test_episodes(seed), ["Base LFM2-350M", "After GRPO"]) @spaces.GPU(duration=75) @browser_session() def book_flight_episodes(seed): episodes = [] for adapter_on in (False, True): # The same tool schema the book-flight evaluations used (bid/value), not the generic default. policy = LFM3Policy(lfm3_base, lfm3_tokenizer, torch, SAMPLED, "native-tools", tools=BOOK_FLIGHT_TOOLS, max_prompt_tokens=16384) with closing(LocalBrowserGymEnvironment.miniwob("book-flight-nodelay", miniwob_url=SERVER.base_url)) as env: if adapter_on: episodes.append(collect_book_flight_episode(env, policy, seed)) else: with lfm3.disable_adapter(): episodes.append(collect_book_flight_episode(env, policy, seed)) return episodes def book_flight(seed): seed = int(seed) return run_text_pair("book-flight-nodelay", seed, book_flight_episodes(seed), ["Base LFM3-1.7B", "With the SFT adapter"]) @spaces.GPU(duration=60) @browser_session() def click_sequence_episodes(seed, decoder_name): decoder = GREEDY if decoder_name == "greedy" else SAMPLED episodes = [] for adapter in (None, "sequence-sft", "sft-then-grpo"): policy = LFM3VisionPolicy(vl_base, vl_processor, torch, decoder) with closing(VisionBrowserGymEnvironment.miniwob("click-button-sequence", miniwob_url=SERVER.base_url)) as env: images = Path(tempfile.mkdtemp()) if adapter is None: with vl.disable_adapter(): episodes.append(collect_vision_sequence(env, policy, seed, "screenshot", images, max_steps=4)) else: vl.set_adapter(adapter) episodes.append(collect_vision_sequence(env, policy, seed, "screenshot", images, max_steps=4)) return episodes def click_sequence(seed, decoder_name): seed = int(seed) labels = ["Base LFM3-VL-2B", "SFT adapter", "SFT, then GRPO"] episodes = click_sequence_episodes(seed, decoder_name) return replay_sequence(seed, decoder_name, labels, episodes) @browser_session() def replay_sequence(seed, decoder_name, labels, episodes): columns, outcomes, notes = [], [], [] for label, episode in zip(labels, episodes): with closing(_ScreenshotCapture.miniwob("click-button-sequence", miniwob_url=SERVER.base_url, coordinate_actions=True)) as env: out = Path(tempfile.mkdtemp()) summary = replay_vision(env, episode, out, 3) columns.append(frames_of(out)) outcomes.append(bool(episode["success"])) notes.append(f"**{label}:** {'success' if episode['success'] else 'failure'} after " f"{episode['environment_steps']} click(s), ended by {episode['stop_reason']}; " f"replay {'matches' if summary['matches'] else 'DIFFERS'}") frames = side_by_side_frames(columns, labels, outcomes, f"click-button-sequence, seed {seed}, {decoder_name}, live") return gif(frames, "click-sequence"), "\n\n".join(notes) INTRO = """ # Base versus tuned, live Pick a task seed, and the base Liquid model and its tuned adapter both attempt that exact task now, side by side. MiniWoB tasks are seeded, so both see the same page. These are **fresh samples on ZeroGPU hardware**, so they vary from run to run and won't match the recorded evaluations token for token. The published numbers come from the locked, recorded runs: see the [replay Space](https://huggingface.co/spaces/Aptheos/browser-control-demo) and the [repository](https://github.com/DerrickKirimi/browser-control). Seeds 21000–22099 and 10100–10299 were held out for evaluation; any seed works here. """ with gr.Blocks(title="Browser control: base vs tuned, live") as demo: gr.Markdown(INTRO) with gr.Tab("Click from pixels (LFM3-VL-2B)"): with gr.Row(): vseed = gr.Number(value=50001, precision=0, label="Task seed") vdec = gr.Radio(["sampled", "greedy"], value="sampled", label="Decoder (GRPO's gain shows under sampling)") vrun = gr.Button("Run all three models", variant="primary") vout, vnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown() vrun.click(click_sequence, [vseed, vdec], [vout, vnote], concurrency_id="browser", concurrency_limit=1) with gr.Tab("Book a flight (LFM3-1.7B)"): with gr.Row(): bseed = gr.Number(value=50001, precision=0, label="Task seed") brun = gr.Button("Run both models", variant="primary") bout, bnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown() brun.click(book_flight, bseed, [bout, bnote], concurrency_id="browser", concurrency_limit=1) with gr.Tab("Click test (LFM2-350M)"): with gr.Row(): cseed = gr.Number(value=50001, precision=0, label="Task seed") crun = gr.Button("Run both models", variant="primary") cout, cnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown() crun.click(click_test, cseed, [cout, cnote], concurrency_id="browser", concurrency_limit=1) if __name__ == "__main__": demo.launch()