cklxx commited on
Commit
3928514
路
verified 路
1 Parent(s): d05eb7b

code: demo recorder

Browse files
Files changed (1) hide show
  1. code/apps/make_demo.py +112 -0
code/apps/make_demo.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Record a short demo video of laya driving a real browser through jev-ultrafast.
2
+
3
+ python apps/make_demo.py out_dir (services: chromium :9222, laya systemone :8791; no text model needed)
4
+
5
+ Runs a few click-only tasks with frame recording, overlays goal / laya's decision / per-step latency, renders MP4 + GIF via ffmpeg.
6
+ """
7
+ import base64, json, os, shutil, subprocess, sys, time
8
+ from pathlib import Path
9
+ sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast")
10
+ os.environ.update(BU_CDP_URL="http://127.0.0.1:9222", TYPESAFE_BASE_URL="http://127.0.0.1:8791", TYPESAFE_API_KEY="local", TEXT_MODEL_API_KEY="local")
11
+ from PIL import Image, ImageDraw, ImageFont
12
+ from jev_ultrafast import Agent
13
+
14
+ TASKS = [
15
+ ("https://books.toscrape.com/", "Open the 'Travel' category."),
16
+ ("https://books.toscrape.com/", "Open the product page of the book 'A Light in the Attic'."),
17
+ ("https://www.python.org/", "Go to the Downloads page."),
18
+ ("https://quotes.toscrape.com/", "Show the quotes tagged 'love'."),
19
+ ("https://news.ycombinator.com/", "Open the 'past' page (front pages from previous days)."),
20
+ ("https://the-internet.herokuapp.com/checkboxes", "Tick the first checkbox."),
21
+ ]
22
+ W, H = 1120, 780; BAR_T, BAR_B = 64, 96; FPS = 24
23
+ F_REG = "/usr/share/fonts/TTF/DejaVuSans.ttf"; F_BOLD = "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf"; F_MONO = "/usr/share/fonts/TTF/DejaVuSansMono.ttf"
24
+ def font(n, bold=False): return ImageFont.truetype(F_BOLD if bold else F_REG, n)
25
+ def mono(n): return ImageFont.truetype(F_MONO, n)
26
+ BG, INK, MUTED, GREEN, BLUE = "#0f1115", "#e8e8e8", "#8b93a1", "#3ddc84", "#4f8cff"
27
+
28
+ def canvas(page_img=None):
29
+ im = Image.new("RGB", (W, BAR_T + H + BAR_B), BG)
30
+ if page_img is not None: im.paste(page_img.resize((W, H)), (0, BAR_T))
31
+ return im
32
+
33
+ def frame(page_img, goal, step_txt, decision_txt, lat_txt, elapsed_txt, status=None):
34
+ im = canvas(page_img); d = ImageDraw.Draw(im)
35
+ d.text((16, 12), "laya-browser", font=font(22, True), fill=GREEN); d.text((190, 16), "System-1 decision head 路 322M 路 local RTX 4070", font=font(15), fill=MUTED)
36
+ d.text((16, 38), "goal: " + goal, font=font(16), fill=INK)
37
+ y = BAR_T + H + 12
38
+ d.text((16, y), step_txt, font=font(15, True), fill=MUTED)
39
+ d.text((16, y + 24), decision_txt, font=mono(19), fill=INK)
40
+ d.text((16, y + 54), lat_txt, font=mono(15), fill=BLUE); d.text((W - 200, y + 54), elapsed_txt, font=mono(15), fill=MUTED)
41
+ if status: d.text((W - 200, y), status, font=font(17, True), fill=GREEN if status == "DONE" else "#ff6b6b")
42
+ return im
43
+
44
+ def title_card(lines, sub):
45
+ im = canvas(); d = ImageDraw.Draw(im); y = 260
46
+ for i, l in enumerate(lines):
47
+ d.text((60, y), l, font=font(40 if i == 0 else 26, i == 0), fill=GREEN if i == 0 else INK); y += 64 if i == 0 else 40
48
+ y += 20
49
+ for l in sub: d.text((60, y), l, font=font(20), fill=MUTED); y += 32
50
+ return im
51
+
52
+ def run_task(url, goal, rec):
53
+ frames, decisions = [], []
54
+ with Agent(url, goal, record_dir=str(rec), screenshots=True) as agent:
55
+ t0 = time.time(); page0 = agent.state["page"]
56
+ frames.append((0, Image.open(rec / "000000.jpg").convert("RGB"), None))
57
+ while agent.state["status"] not in ("done", "blocked") and len(agent.state["history"]) < 8:
58
+ st = agent.command("tick")
59
+ d = st["decisions"][-1] if st["decisions"] else None
60
+ h = st["history"][-1] if st["history"] else None
61
+ lat = d["latency_ms"] if d else 0
62
+ label = (h["action"] if h and d and h.get("latency_ms") == lat else None)
63
+ dec = {"op": d["operation"] if d else "?", "target": label or (d.get("target") if d else ""), "conf": d["confidence"] if d else 0, "lat": lat,
64
+ "elapsed": st["elapsed_ms"], "status": st["status"]}
65
+ decisions.append(dec)
66
+ shots = sorted(p for p in rec.glob("*.jpg") if p.stem != "000000")
67
+ img = Image.open(shots[-1]).convert("RGB") if shots else frames[-1][1]
68
+ frames.append((st["elapsed_ms"], img, dec))
69
+ final = agent.state
70
+ return frames, decisions, final
71
+
72
+ def main():
73
+ out = Path(sys.argv[1]); shutil.rmtree(out, ignore_errors=True); (out / "frames").mkdir(parents=True)
74
+ n = 0
75
+ def emit(im, secs):
76
+ nonlocal n
77
+ for _ in range(int(secs * FPS)):
78
+ im.save(out / "frames" / f"{n:06d}.png"); n += 1
79
+ emit(title_card(["laya-browser", "one bidirectional encoder pass per step -> operation + target + calibrated confidence",
80
+ "no text generation, ~20 ms per decision on an RTX 4070, 1.5 GB VRAM"],
81
+ ["fine-tuned from convaiinnovations/laya on crawled pages + Mind2Web + on-policy corrections",
82
+ "driving browser-use/jev-ultrafast through its TypeSafe-compatible /v1/systemone API"]), 3.5)
83
+ totals = {"steps": 0, "lat": [], "wall": 0.0, "tasks_ok": 0}
84
+ for i, (url, goal) in enumerate(TASKS):
85
+ rec = out / f"rec{i}"; rec.mkdir()
86
+ try:
87
+ t = time.time(); frames, decisions, final = run_task(url, goal, rec); wall = time.time() - t
88
+ except Exception as e:
89
+ print("task failed:", goal, type(e).__name__, str(e)[:60]); continue
90
+ ok = final["status"] == "done"
91
+ totals["steps"] += len(decisions); totals["lat"] += [d["lat"] for d in decisions]; totals["wall"] += wall; totals["tasks_ok"] += ok
92
+ print(f"{goal[:50]:50s} steps={len(decisions)} status={final['status']} wall={wall:.1f}s decisions={[ (d['op'], d['lat']) for d in decisions]}", flush=True)
93
+ # observed page, then each decision: show the page it decided on with the decision, then the resulting page
94
+ emit(frame(frames[0][1], goal, "step 0 路 observe page", "laya reads the element table ...", "", "t = 0 ms"), 1.2)
95
+ for k, (ms, img, dec) in enumerate(frames[1:], 1):
96
+ prev = frames[k - 1][1]
97
+ dtxt = f"{dec['op']} -> {str(dec['target'])[:48]}" if dec["op"] not in ("DONE", "BLOCKED") else dec["op"]
98
+ emit(frame(prev, goal, f"step {k} 路 decide", dtxt, f"laya: {dec['lat']} ms (confidence {dec['conf']:.2f})", f"t = {dec['elapsed']} ms"), 1.3)
99
+ if dec["op"] not in ("DONE", "BLOCKED"):
100
+ emit(frame(img, goal, f"step {k} 路 executed", dtxt, f"laya: {dec['lat']} ms", f"t = {dec['elapsed']} ms"), 1.0)
101
+ else:
102
+ emit(frame(img, goal, f"step {k}", dtxt, f"laya: {dec['lat']} ms", f"t = {dec['elapsed']} ms", status=dec["op"]), 1.6)
103
+ lat = sorted(totals["lat"]); med = lat[len(lat) // 2] if lat else 0
104
+ emit(title_card(["that's laya as a browser agent's System 1", f"{totals['tasks_ok']}/{len(TASKS)} tasks 路 {totals['steps']} decisions 路 median {med} ms per decision",
105
+ f"total wall time {totals['wall']:.1f} s including page loads"],
106
+ ["model + code + numbers: huggingface.co/cklxx/laya-browser", "TileLang fused kernels + CUDA graphs: github.com/NandhaKishorM/laya/pull/25"]), 4.0)
107
+ subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-framerate", str(FPS), "-i", str(out / "frames" / "%06d.png"), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "23", str(out / "laya_browser_demo.mp4")], check=True)
108
+ subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", str(out / "laya_browser_demo.mp4"), "-vf", "fps=8,scale=700:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=128[p];[s1][p]paletteuse=dither=bayer", str(out / "laya_browser_demo.gif")], check=True)
109
+ print("video:", out / "laya_browser_demo.mp4", f"{(out / 'laya_browser_demo.mp4').stat().st_size/1e6:.1f} MB", "| gif:", f"{(out / 'laya_browser_demo.gif').stat().st_size/1e6:.1f} MB", "| frames", n, f"({n/FPS:.0f}s)")
110
+
111
+ if __name__ == "__main__":
112
+ main()