Aptheos's picture
Isolate Playwright connections per ZeroGPU worker and serialize browser requests
975d017 verified
Raw History Blame Contribute Delete
12.2 kB
"""Live side-by-side: base Liquid models and their tuned adapters on the same seeded MiniWoB task.
Each request runs the base model and the tuned model on one task seed you choose, with the
repository's own prompts, parsers and episode loops, then replays both episodes into frames. These
are fresh samples on ZeroGPU hardware, not the locked evaluations: the published numbers come from the
recorded runs linked below.
"""
import subprocess
import sys
import tempfile
from contextlib import closing
from pathlib import Path
HERE = Path(__file__).parent
sys.path.insert(0, str(HERE))
subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"], check=True)
import gradio as gr # noqa: E402
import spaces # noqa: E402
import torch # noqa: E402
from peft import PeftModel # noqa: E402
from PIL import Image # noqa: E402
from transformers import ( # noqa: E402
AutoModelForCausalLM,
AutoModelForImageTextToText,
AutoProcessor,
AutoTokenizer,
GenerationConfig,
)
from browser_control.book_flight import BOOK_FLIGHT_TOOLS # noqa: E402
from browser_control.book_flight_evaluation import collect_book_flight_episode # noqa: E402
from browser_control.evaluation import evaluate_click_test # noqa: E402
from browser_control.lfm3 import LFM3Policy # noqa: E402
from browser_control.lfm3_vision import LFM3VisionPolicy # noqa: E402
from browser_control.local_browsergym import LocalBrowserGymEnvironment # noqa: E402
from browser_control.miniwob_assets import MiniwobAssetServer, ensure_miniwob_assets # noqa: E402
from browser_control.replay import side_by_side_frames # noqa: E402
from browser_control.transformers_policy import TRAINING_DECODER, DecoderConfig, TransformersPolicy # noqa: E402
from browser_control.vision_browser import VisionBrowserGymEnvironment, _ScreenshotCapture # noqa: E402
from browser_control.vision_evaluation import collect_vision_sequence # noqa: E402
from replay_tools import replay, replay_vision, single_step # noqa: E402
from browser_session import browser_session # noqa: E402
# Copies of the published adapters (same bytes as the model repos), so the Space needs no token.
ADAPTERS = HERE / "adapters"
LFM3_REVISION = "ac2586a34f1a1c03329dbdcdefabd8b6478ddaa7"
VL_REVISION = "ba44bd51db20a2753f28c273a1571ccaa210d954"
SAMPLED = DecoderConfig("sampling", True, 128, temperature=1.0, top_p=0.95)
GREEDY = DecoderConfig("greedy", False, 128)
EXPLICIT = dict(top_k=0, repetition_penalty=1.0)
def plain_generation_config(model, tokenizer):
"""Don't inherit checkpoint sampling defaults; the decoder is set per call, as in evaluation."""
model.generation_config = GenerationConfig(
bos_token_id=tokenizer.bos_token_id, eos_token_id=tokenizer.eos_token_id,
pad_token_id=tokenizer.pad_token_id, **EXPLICIT)
class Device:
index = 0
# LFM2-350M with its GRPO adapter (seed 23).
lfm2_tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-350M")
lfm2_base = AutoModelForCausalLM.from_pretrained("LiquidAI/LFM2-350M", dtype=torch.float16).eval()
lfm2 = PeftModel.from_pretrained(lfm2_base, str(ADAPTERS / "lfm2-click-test"), torch_device="cpu").to("cuda").eval()
# LFM3-1.7B with its book-flight SFT adapter.
lfm3_tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM3-1.7B", revision=LFM3_REVISION, trust_remote_code=True, padding_side="left")
lfm3_base = AutoModelForCausalLM.from_pretrained(
"LiquidAI/LFM3-1.7B", revision=LFM3_REVISION, trust_remote_code=True, dtype=torch.float16, attn_implementation="sdpa"
).eval()
plain_generation_config(lfm3_base, lfm3_tokenizer)
lfm3 = PeftModel.from_pretrained(lfm3_base, str(ADAPTERS / "lfm3-book-flight"), torch_device="cpu").to("cuda").eval()
# LFM3-VL-2B with the click-sequence SFT adapter (and the click-color one it started from).
vl_processor = AutoProcessor.from_pretrained("LiquidAI/LFM3-VL-2B", revision=VL_REVISION, trust_remote_code=True)
vl_base = AutoModelForImageTextToText.from_pretrained(
"LiquidAI/LFM3-VL-2B", revision=VL_REVISION, trust_remote_code=True, dtype=torch.float16, attn_implementation="sdpa"
)
vl = PeftModel.from_pretrained(vl_base, str(ADAPTERS / "lfm3-vl-sequence"), adapter_name="sequence-sft", torch_device="cpu")
vl.load_adapter(str(ADAPTERS / "lfm3-vl-grpo"), adapter_name="sft-then-grpo", torch_device="cpu")
vl = vl.to("cuda").eval()
MINIWOB_ROOT = ensure_miniwob_assets(HERE / "miniwob-plusplus")
SERVER = MiniwobAssetServer(MINIWOB_ROOT)
SERVER.__enter__()
def gif(frames, title):
path = Path(tempfile.mkdtemp()) / "side-by-side.gif"
palette = [f.quantize(colors=256, method=Image.Quantize.MEDIANCUT, dither=Image.Dither.NONE) for f in frames]
palette[0].save(path, save_all=True, append_images=palette[1:], duration=[1400] * (len(frames) - 1) + [3200],
loop=0, optimize=True)
return str(path)
def frames_of(directory):
return [Image.open(p).convert("RGB") for p in sorted(Path(directory).glob("frame-*.png"))]
@browser_session()
def run_text_pair(task, seed, episodes, labels):
"""Replay each recorded episode in the seeded task and compose them side by side."""
columns, outcomes, notes = [], [], []
with closing(_ScreenshotCapture.miniwob(task, miniwob_url=SERVER.base_url)) as env:
for episode, label in zip(episodes, labels):
out = Path(tempfile.mkdtemp())
summary = replay(env, episode, out, 1, 3)
columns.append(frames_of(out))
outcomes.append(bool(episode.get("success")))
last = (episode["steps"][-1].get("text") or "") if episode["steps"] else ""
notes.append(f"**{label}:** {'success' if episode.get('success') else 'failure'}, "
f"{len(episode['steps'])} step(s), ended by {episode.get('stop_reason', 'done')}; "
f"replay {'matches' if summary['matches'] else 'DIFFERS'}\n\nLast output: `{' '.join(last.split())[:300]}`")
frames = side_by_side_frames(columns, labels, outcomes, f"{task}, seed {seed}, live samples")
return gif(frames, task), "\n\n".join(notes)
# GPU time is spent only on model inference; episodes are replayed and drawn afterwards on the CPU.
@spaces.GPU(duration=20)
@browser_session()
def click_test_episodes(seed):
episodes = []
for adapter_on in (False, True):
policy = TransformersPolicy(model=lfm2, tokenizer=lfm2_tokenizer, torch=torch, device=Device(), decoder=TRAINING_DECODER)
factory = lambda: LocalBrowserGymEnvironment.miniwob("click-test", miniwob_url=SERVER.base_url) # noqa: E731
if adapter_on:
report = evaluate_click_test(factory, policy, [seed])
else:
with lfm2.disable_adapter():
report = evaluate_click_test(factory, policy, [seed])
episodes.append(single_step(report.to_dict()["episodes"][0]))
return episodes
def click_test(seed):
seed = int(seed)
return run_text_pair("click-test", seed, click_test_episodes(seed), ["Base LFM2-350M", "After GRPO"])
@spaces.GPU(duration=75)
@browser_session()
def book_flight_episodes(seed):
episodes = []
for adapter_on in (False, True):
# The same tool schema the book-flight evaluations used (bid/value), not the generic default.
policy = LFM3Policy(lfm3_base, lfm3_tokenizer, torch, SAMPLED, "native-tools", tools=BOOK_FLIGHT_TOOLS,
max_prompt_tokens=16384)
with closing(LocalBrowserGymEnvironment.miniwob("book-flight-nodelay", miniwob_url=SERVER.base_url)) as env:
if adapter_on:
episodes.append(collect_book_flight_episode(env, policy, seed))
else:
with lfm3.disable_adapter():
episodes.append(collect_book_flight_episode(env, policy, seed))
return episodes
def book_flight(seed):
seed = int(seed)
return run_text_pair("book-flight-nodelay", seed, book_flight_episodes(seed), ["Base LFM3-1.7B", "With the SFT adapter"])
@spaces.GPU(duration=60)
@browser_session()
def click_sequence_episodes(seed, decoder_name):
decoder = GREEDY if decoder_name == "greedy" else SAMPLED
episodes = []
for adapter in (None, "sequence-sft", "sft-then-grpo"):
policy = LFM3VisionPolicy(vl_base, vl_processor, torch, decoder)
with closing(VisionBrowserGymEnvironment.miniwob("click-button-sequence", miniwob_url=SERVER.base_url)) as env:
images = Path(tempfile.mkdtemp())
if adapter is None:
with vl.disable_adapter():
episodes.append(collect_vision_sequence(env, policy, seed, "screenshot", images, max_steps=4))
else:
vl.set_adapter(adapter)
episodes.append(collect_vision_sequence(env, policy, seed, "screenshot", images, max_steps=4))
return episodes
def click_sequence(seed, decoder_name):
seed = int(seed)
labels = ["Base LFM3-VL-2B", "SFT adapter", "SFT, then GRPO"]
episodes = click_sequence_episodes(seed, decoder_name)
return replay_sequence(seed, decoder_name, labels, episodes)
@browser_session()
def replay_sequence(seed, decoder_name, labels, episodes):
columns, outcomes, notes = [], [], []
for label, episode in zip(labels, episodes):
with closing(_ScreenshotCapture.miniwob("click-button-sequence", miniwob_url=SERVER.base_url,
coordinate_actions=True)) as env:
out = Path(tempfile.mkdtemp())
summary = replay_vision(env, episode, out, 3)
columns.append(frames_of(out))
outcomes.append(bool(episode["success"]))
notes.append(f"**{label}:** {'success' if episode['success'] else 'failure'} after "
f"{episode['environment_steps']} click(s), ended by {episode['stop_reason']}; "
f"replay {'matches' if summary['matches'] else 'DIFFERS'}")
frames = side_by_side_frames(columns, labels, outcomes, f"click-button-sequence, seed {seed}, {decoder_name}, live")
return gif(frames, "click-sequence"), "\n\n".join(notes)
INTRO = """
# Base versus tuned, live
Pick a task seed, and the base Liquid model and its tuned adapter both attempt that exact task now, side by side.
MiniWoB tasks are seeded, so both see the same page. These are **fresh samples on ZeroGPU hardware**, so they vary from
run to run and won't match the recorded evaluations token for token. The published numbers come from the locked,
recorded runs: see the [replay Space](https://huggingface.co/spaces/Aptheos/browser-control-demo) and the
[repository](https://github.com/DerrickKirimi/browser-control). Seeds 21000–22099 and 10100–10299 were held out for
evaluation; any seed works here.
"""
with gr.Blocks(title="Browser control: base vs tuned, live") as demo:
gr.Markdown(INTRO)
with gr.Tab("Click from pixels (LFM3-VL-2B)"):
with gr.Row():
vseed = gr.Number(value=50001, precision=0, label="Task seed")
vdec = gr.Radio(["sampled", "greedy"], value="sampled", label="Decoder (GRPO's gain shows under sampling)")
vrun = gr.Button("Run all three models", variant="primary")
vout, vnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown()
vrun.click(click_sequence, [vseed, vdec], [vout, vnote], concurrency_id="browser", concurrency_limit=1)
with gr.Tab("Book a flight (LFM3-1.7B)"):
with gr.Row():
bseed = gr.Number(value=50001, precision=0, label="Task seed")
brun = gr.Button("Run both models", variant="primary")
bout, bnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown()
brun.click(book_flight, bseed, [bout, bnote], concurrency_id="browser", concurrency_limit=1)
with gr.Tab("Click test (LFM2-350M)"):
with gr.Row():
cseed = gr.Number(value=50001, precision=0, label="Task seed")
crun = gr.Button("Run both models", variant="primary")
cout, cnote = gr.Image(label="Side by side", type="filepath"), gr.Markdown()
crun.click(click_test, cseed, [cout, cnote], concurrency_id="browser", concurrency_limit=1)
if __name__ == "__main__":
demo.launch()