Spaces:
Running on Zero
Running on Zero
Download app.py from Aptheos/browser-control-live: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/spaces/Aptheos/browser-control-live/resolve/main/app.py
- Command line
-
hf download hf://spaces/Aptheos/browser-control-live/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Aptheos/browser-control-live/resolve/main/app.py
12.2 kB
| """Live side-by-side: base Liquid models and their tuned adapters on the same seeded MiniWoB task. | |
| Each request runs the base model and the tuned model on one task seed you choose, with the | |
| repository's own prompts, parsers and episode loops, then replays both episodes into frames. These | |
| are fresh samples on ZeroGPU hardware, not the locked evaluations: the published numbers come from the | |
| recorded runs linked below. | |
| """ | |
| import subprocess | |
| import sys | |
| import tempfile | |
| from contextlib import closing | |
| from pathlib import Path | |
| HERE = Path(__file__).parent | |
| sys.path.insert(0, str(HERE)) | |
| subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"], check=True) | |
| import gradio as gr # noqa: E402 | |
| import spaces # noqa: E402 | |
| import torch # noqa: E402 | |
| from peft import PeftModel # noqa: E402 | |
| from PIL import Image # noqa: E402 | |
| from transformers import ( # noqa: E402 | |
| AutoModelForCausalLM, | |
| AutoModelForImageTextToText, | |
| AutoProcessor, | |
| AutoTokenizer, | |
| GenerationConfig, | |
| ) | |
| from browser_control.book_flight import BOOK_FLIGHT_TOOLS # noqa: E402 | |
| from browser_control.book_flight_evaluation import collect_book_flight_episode # noqa: E402 | |
| from browser_control.evaluation import evaluate_click_test # noqa: E402 | |
| from browser_control.lfm3 import LFM3Policy # noqa: E402 | |
| from browser_control.lfm3_vision import LFM3VisionPolicy # noqa: E402 | |
| from browser_control.local_browsergym import LocalBrowserGymEnvironment # noqa: E402 | |
| from browser_control.miniwob_assets import MiniwobAssetServer, ensure_miniwob_assets # noqa: E402 | |
| from browser_control.replay import side_by_side_frames # noqa: E402 | |
| from browser_control.transformers_policy import TRAINING_DECODER, DecoderConfig, TransformersPolicy # noqa: E402 | |
| from browser_control.vision_browser import VisionBrowserGymEnvironment, _ScreenshotCapture # noqa: E402 | |
| from browser_control.vision_evaluation import collect_vision_sequence # noqa: E402 | |
| from replay_tools import replay, replay_vision, single_step # noqa: E402 | |
| from browser_session import browser_session # noqa: E402 | |
| # Copies of the published adapters (same bytes as the model repos), so the Space needs no token. | |
| ADAPTERS = HERE / "adapters" | |
| LFM3_REVISION = "ac2586a34f1a1c03329dbdcdefabd8b6478ddaa7" | |
| VL_REVISION = "ba44bd51db20a2753f28c273a1571ccaa210d954" | |
| SAMPLED = DecoderConfig("sampling", True, 128, temperature=1.0, top_p=0.95) | |
| GREEDY = DecoderConfig("greedy", False, 128) | |
| EXPLICIT = dict(top_k=0, repetition_penalty=1.0) | |
| def plain_generation_config(model, tokenizer): | |
| """Don't inherit checkpoint sampling defaults; the decoder is set per call, as in evaluation.""" | |
| model.generation_config = GenerationConfig( | |
| bos_token_id=tokenizer.bos_token_id, eos_token_id=tokenizer.eos_token_id, | |
| pad_token_id=tokenizer.pad_token_id, **EXPLICIT) | |
| class Device: | |
| index = 0 | |
| # LFM2-350M with its GRPO adapter (seed 23). | |
| lfm2_tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-350M") | |
| lfm2_base = AutoModelForCausalLM.from_pretrained("LiquidAI/LFM2-350M", dtype=torch.float16).eval() | |
| lfm2 = PeftModel.from_pretrained(lfm2_base, str(ADAPTERS / "lfm2-click-test"), torch_device="cpu").to("cuda").eval() | |
| # LFM3-1.7B with its book-flight SFT adapter. | |
| lfm3_tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM3-1.7B", revision=LFM3_REVISION, trust_remote_code=True, padding_side="left") | |
| lfm3_base = AutoModelForCausalLM.from_pretrained( | |
| "LiquidAI/LFM3-1.7B", revision=LFM3_REVISION, trust_remote_code=True, dtype=torch.float16, attn_implementation="sdpa" | |
| ).eval() | |
| plain_generation_config(lfm3_base, lfm3_tokenizer) | |
| lfm3 = PeftModel.from_pretrained(lfm3_base, str(ADAPTERS / "lfm3-book-flight"), torch_device="cpu").to("cuda").eval() | |
| # LFM3-VL-2B with the click-sequence SFT adapter (and the click-color one it started from). | |
| vl_processor = AutoProcessor.from_pretrained("LiquidAI/LFM3-VL-2B", revision=VL_REVISION, trust_remote_code=True) | |
| vl_base = AutoModelForImageTextToText.from_pretrained( | |
| "LiquidAI/LFM3-VL-2B", revision=VL_REVISION, trust_remote_code=True, dtype=torch.float16, attn_implementation="sdpa" | |
| ) | |
| vl = PeftModel.from_pretrained(vl_base, str(ADAPTERS / "lfm3-vl-sequence"), adapter_name="sequence-sft", torch_device="cpu") | |
| vl.load_adapter(str(ADAPTERS / "lfm3-vl-grpo"), adapter_name="sft-then-grpo", torch_device="cpu") | |
| vl = vl.to("cuda").eval() | |
| MINIWOB_ROOT = ensure_miniwob_assets(HERE / "miniwob-plusplus") | |
| SERVER = MiniwobAssetServer(MINIWOB_ROOT) | |
| SERVER.__enter__() | |
| def gif(frames, title): | |
| path = Path(tempfile.mkdtemp()) / "side-by-side.gif" | |
| palette = [f.quantize(colors=256, method=Image.Quantize.MEDIANCUT, dither=Image.Dither.NONE) for f in frames] | |
| palette[0].save(path, save_all=True, append_images=palette[1:], duration=[1400] * (len(frames) - 1) + [3200], | |
| loop=0, optimize=True) | |
| return str(path) | |
| def frames_of(directory): | |
| return [Image.open(p).convert("RGB") for p in sorted(Path(directory).glob("frame-*.png"))] | |
| def run_text_pair(task, seed, episodes, labels): | |
| """Replay each recorded episode in the seeded task and compose them side by side.""" | |
| columns, outcomes, notes = [], [], [] | |
| with closing(_ScreenshotCapture.miniwob(task, miniwob_url=SERVER.base_url)) as env: | |
| for episode, label in zip(episodes, labels): | |
| out = Path(tempfile.mkdtemp()) | |
| summary = replay(env, episode, out, 1, 3) | |
| columns.append(frames_of(out)) | |
| outcomes.append(bool(episode.get("success"))) | |
| last = (episode["steps"][-1].get("text") or "") if episode["steps"] else "" | |
| notes.append(f"**{label}:** {'success' if episode.get('success') else 'failure'}, " | |
| f"{len(episode['steps'])} step(s), ended by {episode.get('stop_reason', 'done')}; " | |
| f"replay {'matches' if summary['matches'] else 'DIFFERS'}\n\nLast output: `{' '.join(last.split())[:300]}`") | |
| frames = side_by_side_frames(columns, labels, outcomes, f"{task}, seed {seed}, live samples") | |
| return gif(frames, task), "\n\n".join(notes) | |
| # GPU time is spent only on model inference; episodes are replayed and drawn afterwards on the CPU. | |
| def click_test_episodes(seed): | |
| episodes = [] | |
| for adapter_on in (False, True): | |
| policy = TransformersPolicy(model=lfm2, tokenizer=lfm2_tokenizer, torch=torch, device=Device(), decoder=TRAINING_DECODER) | |
| factory = lambda: LocalBrowserGymEnvironment.miniwob("click-test", miniwob_url=SERVER.base_url) # noqa: E731 | |
| if adapter_on: | |
| report = evaluate_click_test(factory, policy, [seed]) | |
| else: | |
| with lfm2.disable_adapter(): | |
| report = evaluate_click_test(factory, policy, [seed]) | |
| episodes.append(single_step(report.to_dict()["episodes"][0])) | |
| return episodes | |
| def click_test(seed): | |
| seed = int(seed) | |
| return run_text_pair("click-test", seed, click_test_episodes(seed), ["Base LFM2-350M", "After GRPO"]) | |
| def book_flight_episodes(seed): | |
| episodes = [] | |
| for adapter_on in (False, True): | |
| # The same tool schema the book-flight evaluations used (bid/value), not the generic default. | |
| policy = LFM3Policy(lfm3_base, lfm3_tokenizer, torch, SAMPLED, "native-tools", tools=BOOK_FLIGHT_TOOLS, | |
| max_prompt_tokens=16384) | |
| with closing(LocalBrowserGymEnvironment.miniwob("book-flight-nodelay", miniwob_url=SERVER.base_url)) as env: | |
| if adapter_on: | |
| episodes.append(collect_book_flight_episode(env, policy, seed)) | |
| else: | |
| with lfm3.disable_adapter(): | |
| episodes.append(collect_book_flight_episode(env, policy, seed)) | |
| return episodes | |
| def book_flight(seed): | |
| seed = int(seed) | |
| return run_text_pair("book-flight-nodelay", seed, book_flight_episodes(seed), ["Base LFM3-1.7B", "With the SFT adapter"]) | |
| def click_sequence_episodes(seed, decoder_name): | |
| decoder = GREEDY if decoder_name == "greedy" else SAMPLED | |
| episodes = [] | |
| for adapter in (None, "sequence-sft", "sft-then-grpo"): | |
| policy = LFM3VisionPolicy(vl_base, vl_processor, torch, decoder) | |
| with closing(VisionBrowserGymEnvironment.miniwob("click-button-sequence", miniwob_url=SERVER.base_url)) as env: | |
| images = Path(tempfile.mkdtemp()) | |
| if adapter is None: | |
| with vl.disable_adapter(): | |
| episodes.append(collect_vision_sequence(env, policy, seed, "screenshot", images, max_steps=4)) | |
| else: | |
| vl.set_adapter(adapter) | |
| episodes.append(collect_vision_sequence(env, policy, seed, "screenshot", images, max_steps=4)) | |
| return episodes | |
| def click_sequence(seed, decoder_name): | |
| seed = int(seed) | |
| labels = ["Base LFM3-VL-2B", "SFT adapter", "SFT, then GRPO"] | |
| episodes = click_sequence_episodes(seed, decoder_name) | |
| return replay_sequence(seed, decoder_name, labels, episodes) | |
| def replay_sequence(seed, decoder_name, labels, episodes): | |
| columns, outcomes, notes = [], [], [] | |
| for label, episode in zip(labels, episodes): | |
| with closing(_ScreenshotCapture.miniwob("click-button-sequence", miniwob_url=SERVER.base_url, | |
| coordinate_actions=True)) as env: | |
| out = Path(tempfile.mkdtemp()) | |
| summary = replay_vision(env, episode, out, 3) | |
| columns.append(frames_of(out)) | |
| outcomes.append(bool(episode["success"])) | |
| notes.append(f"**{label}:** {'success' if episode['success'] else 'failure'} after " | |
| f"{episode['environment_steps']} click(s), ended by {episode['stop_reason']}; " | |
| f"replay {'matches' if summary['matches'] else 'DIFFERS'}") | |
| frames = side_by_side_frames(columns, labels, outcomes, f"click-button-sequence, seed {seed}, {decoder_name}, live") | |
| return gif(frames, "click-sequence"), "\n\n".join(notes) | |
| INTRO = """ | |
| # Base versus tuned, live | |
| Pick a task seed, and the base Liquid model and its tuned adapter both attempt that exact task now, side by side. | |
| MiniWoB tasks are seeded, so both see the same page. These are **fresh samples on ZeroGPU hardware**, so they vary from | |
| run to run and won't match the recorded evaluations token for token. The published numbers come from the locked, | |
| recorded runs: see the [replay Space](https://huggingface.co/spaces/Aptheos/browser-control-demo) and the | |
| [repository](https://github.com/DerrickKirimi/browser-control). Seeds 21000–22099 and 10100–10299 were held out for | |
| evaluation; any seed works here. | |
| """ | |
| with gr.Blocks(title="Browser control: base vs tuned, live") as demo: | |
| gr.Markdown(INTRO) | |
| with gr.Tab("Click from pixels (LFM3-VL-2B)"): | |
| with gr.Row(): | |
| vseed = gr.Number(value=50001, precision=0, label="Task seed") | |
| vdec = gr.Radio(["sampled", "greedy"], value="sampled", label="Decoder (GRPO's gain shows under sampling)") | |
| vrun = gr.Button("Run all three models", variant="primary") | |
| vout, vnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown() | |
| vrun.click(click_sequence, [vseed, vdec], [vout, vnote], concurrency_id="browser", concurrency_limit=1) | |
| with gr.Tab("Book a flight (LFM3-1.7B)"): | |
| with gr.Row(): | |
| bseed = gr.Number(value=50001, precision=0, label="Task seed") | |
| brun = gr.Button("Run both models", variant="primary") | |
| bout, bnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown() | |
| brun.click(book_flight, bseed, [bout, bnote], concurrency_id="browser", concurrency_limit=1) | |
| with gr.Tab("Click test (LFM2-350M)"): | |
| with gr.Row(): | |
| cseed = gr.Number(value=50001, precision=0, label="Task seed") | |
| crun = gr.Button("Run both models", variant="primary") | |
| cout, cnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown() | |
| crun.click(click_test, cseed, [cout, cnote], concurrency_id="browser", concurrency_limit=1) | |
| if __name__ == "__main__": | |
| demo.launch() | |