File size: 19,950 Bytes
ba80edb
 
 
 
 
 
 
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0abcf88
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f321127
 
0df70b6
29b362f
 
 
 
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
f321127
 
 
 
 
 
29b362f
 
0df70b6
f321127
0df70b6
29b362f
 
f321127
29b362f
 
 
 
 
f321127
 
29b362f
 
f321127
 
29b362f
 
 
f321127
 
29b362f
 
 
 
 
f321127
 
29b362f
f321127
29b362f
0df70b6
 
29b362f
 
f321127
 
29b362f
0df70b6
 
29b362f
 
 
 
 
 
 
 
 
 
 
 
 
f321127
29b362f
 
f321127
 
0df70b6
 
f321127
 
 
29b362f
0df70b6
 
f321127
0df70b6
f321127
 
 
 
29b362f
 
f321127
0df70b6
f321127
29b362f
 
0df70b6
f321127
 
0df70b6
 
29b362f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0abcf88
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f321127
 
 
 
 
 
 
29b362f
f321127
 
 
29b362f
f321127
 
0df70b6
f321127
0df70b6
 
 
f321127
 
29b362f
0df70b6
 
29b362f
 
 
0df70b6
f321127
0df70b6
f321127
 
29b362f
 
f321127
29b362f
 
 
 
 
f321127
0df70b6
f321127
0df70b6
29b362f
 
 
 
0df70b6
29b362f
 
 
 
0df70b6
 
 
f321127
29b362f
 
 
0abcf88
29b362f
f321127
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29b362f
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ba80edb
 
 
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ba80edb
 
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0abcf88
 
 
0df70b6
 
 
 
 
 
 
 
 
 
 
 
 
0abcf88
0df70b6
 
 
 
 
 
 
 
 
0abcf88
0df70b6
 
 
 
 
 
 
 
 
 
 
 
ba80edb
 
 
 
 
 
 
 
 
 
 
 
 
 
0df70b6
ba80edb
 
0df70b6
 
ba80edb
0df70b6
 
ba80edb
 
0df70b6
 
 
ba80edb
 
0df70b6
ba80edb
 
 
0df70b6
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
# /// script
# requires-python = ">=3.13"
# dependencies = [
#     "rapidocr-onnxruntime==1.4.4",
# ]
# ///

import marimo

__generated_with = "0.25.2"
app = marimo.App(width="medium", auto_download=["html"])

with app.setup:
    import os
    from pathlib import Path

    import marimo as mo
    from huggingface_hub import HfApi



@app.cell(hide_code=True)
def readme():
    readme_md = '''

    ---
    title: Tiny Computer Agent
    emoji: πŸ–±οΈ
    colorFrom: indigo
    colorTo: green
    license: mit
    tags:
      - computer-use
      - gui-agent
      - agentic-harness
      - local-first
      - tiny-models
      - offline
      - marimo
    ---

    # πŸ–±οΈ Tiny Computer Agent

    **An agentic harness that drives a real computer with a tiny, fully local model.**
    No cloud API, no API keys, no telemetry β€” and a core that imports nothing outside the
    Python standard library.

    > **Status 0.3.0: it drives a real desktop.** Screen capture, an OCR element index with
    > real coordinates, a real mouse click, and a *verified* outcome β€” all through the
    > harness, in one episode. What is not solved is model reliability: see
    > [measured results](#measured-results).

    ## The idea, in one loop

    ```
        observe         think          act           verify
      +---------+    +---------+    +---------+    +---------+
      | screen  |--->|  tiny   |--->| mouse / |--->|  next   |
      |  frame  |    |  model  |    | keyboard|    |  frame  |
      +---------+    +---------+    +---------+    +----+----+
                                                        |
      <-------------------------------------------------+
            repeat until done, or the step budget runs out
    ```

    1. **Observe** β€” capture the screen, then build an **element index**: OCR (and,
       optionally, the OS accessibility tree) locates text and widgets and gives each one a
       label and a real screen coordinate.
    2. **Think** β€” the model is shown a *short numbered menu* built from that index and
       answers with one number (plus a verb). It never emits coordinates.
    3. **Act** β€” the harness maps the chosen label back to pixels and executes it.
    4. **Verify** β€” re-read the screen; stop if the same action repeats against an unchanged
       screen, and only call the episode a success if a verifier can see it worked.

    ## Measured results

    Everything below was run from this repo on CPU, dry-run unless stated. Nothing is
    aspirational.

    ### On a live desktop, with a real click

    A real Tk window (a Save dialog with Save/Cancel buttons) on a **real X server**
    (`Xvfb :99`), driven with real mouse input. The button sits at window `+120+120`,
    `x=20, y=155`, `110x40`, so its centre is `(195, 295)`:

    ```
    $ python -m tiny_computer_agent --live --real-input --verify-text Saved "click 'Save' to save the document"
    mode: REAL INPUT (mouse and keyboard live)  |  screen: real desktop (monitor 1)  |  brain: planner
    index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline))

      step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (195, 296)

    result: success - verified after 1 action(s)
    ```

    The click landed on the button, the app changed its own window to "Saved", and the
    verifier confirmed it before the episode was called a success. **Pixels in, a real click
    out, and evidence that it worked.**

    ### On a replayed screenshot

    ```
    $ python -m tiny_computer_agent --image example --brain planner
    elements detected: 5
      step 0 | click label='Save' ... -> pixel (295, 284)          # OCR found the button
    ```

    **SmolVLM-256M-Instruct** (256M parameters, CPU) read the button off the pixels and
    named the right target:

    ```
      step 0 | click label='Save' reason="the reply names 'Save'" -> pixel (295, 284)
    ```

    ### What did *not* work, and what caught it

    - **The 256M VLM echoes the prompt on a busy live screen.** Asked to click Save on the
      live Tk window it replied with the prompt text back. The harness's echo detection
      refused it (`BrainError: the reply mentions 4 menu entries, so it looks like an echo of
      the prompt`) and stopped β€” **no blind click**. On the cleaner synthetic screenshot the
      same model chose correctly, so this is a prompt/robustness problem, not a wiring one.
    - **Qwen2.5-0.5B text planner over the OCR index is not reliable**: it clicked Cancel
      before Save, then recovered a garbled JSON reply, then found Save β€” budget exhausted.
    - **The grid fallback announced itself.** After the dialog closed, OCR found no text, the
      index degraded to 96 numbered cells, and the run printed
      `warning: no text was found on screen, so the index is 96 numbered grid cells and the
      agent is choosing blind` β€” instead of silently guessing.

    **123 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests
    that click a real button.

    ## Install

    ```bash
    pip install -e ".[ocr]"              # labels from pixels (RapidOCR: no system binary)
    pip install -e ".[desktop,ocr]"      # + real capture and real input
    pip install -e ".[vision,ocr]"       # + the tiny model brains
    pytest                               # 123 tests, no display or network needed
    ```

    ## Run it on your machine

    ```bash
    python -m tiny_computer_agent --doctor                 # what can this machine do?
    python -m tiny_computer_agent --image example          # replay a screenshot, dry run
    python -m tiny_computer_agent --live                   # your real screen, dry run
    python -m tiny_computer_agent --live --real-input "click 'OK'"
    python -m tiny_computer_agent --live --real-input --verify-text "Saved" "click 'Save'"
    ```

    `--doctor` is the first thing to run on a new machine: it checks the display session,
    capture backend, pyautogui, the OCR engine, the model runtime and the kill switch, and
    prints the fix for anything missing. Real input needs **two** deliberate acts:

    ```bash
    export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1
    ```

    **Pass `--verify-text`.** Without it, a successful action that removes its own target
    (clicking Save closes the dialog) is indistinguishable from a failure, and the episode is
    recorded as one. With it, the harness only reports success when it can see the text it
    was given.

    ### No desktop? Use a virtual one

    Xvfb is enough for everything except a real model's hallucinations:

    ```bash
    Xvfb :99 -screen 0 1280x800x24 &
    export DISPLAY=:99
    TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py
    ```

    That test starts its own Tk window, asserts OCR finds the button at the coordinates it
    was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in
    because it moves your mouse β€” run it on a throwaway desktop.

    ### Testing on Windows (or macOS)

    This repo's live test was run on Linux/X11, because that is what the build container
    had. For real Windows coverage, mirror the repo to GitHub: `.github/workflows/live-desktop.yml`
    runs the unit suite **and** the live desktop test on `windows-latest` β€” a real Windows
    desktop session, free for public repositories β€” plus the same test under Xvfb on Linux,
    and a preflight on macOS. Trigger it from the Actions tab.

    Nothing in the harness is Linux-specific: pyautogui uses `SendInput` on Windows, and
    `GrabScreen` already falls back to Pillow's `ImageGrab` there. Two things can only fail
    on a real Windows desktop, which is exactly why the job exists: **DPI scaling** (that is
    what `--scale` is for) and the native input backend. At 125% or 150% display scaling,
    expect the coordinate assertion in `test_live_desktop.py` to be the first thing that
    fails β€” that is the signal, not a nuisance.

    ## How it works

    | Module | Job |
    |---|---|
    | `actions.py` | The closed vocabulary β€” `click` `type` `key` `scroll` `wait` `done`, validated before anything executes |
    | `observe.py` | `Element`/`Observation`, `FakeScreen`, `ImageScreen`, `GrabScreen` (mss β†’ Pillow fallback) |
    | `perception.py` | Detectors: `OcrDetector` (RapidOCR), `GridDetector`, `AutoDetector`, `IndexedScreen` |
    | `brain.py` | `DecisionMenu` (the numbered menu), `ModelBrain`, lenient `parse_decision`, echo detection, `MockBrain`, `PlannerBrain` |
    | `models.py` | `VLMBrain` (SmolVLM-256M) and `TextPlannerBrain` (Qwen2.5-0.5B), lazily loaded |
    | `act.py` | `DryRunExecutor` (the default) and `PyAutoGuiExecutor` |
    | `safety.py` | `Policy`: deny lists, kill switch, step budget, the blind-grid-click guard |
    | `loop.py` | `AgentLoop`, `Episode`, `Step`, the no-progress guard and the verifier |
    | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call |
    | `demo.py` | The synthetic desktop used by tests and `--image example` |

    ## Why a tiny model can do this at all

    | Job | Who does it |
    |---|---|
    | Finding *where* a button is | OCR / accessibility tree β€” **not** the model |
    | Choosing *what* to do | the model, but only by picking a number from a short menu |
    | Knowing whether it worked | the verifier and the no-progress guard, not the model's word |
    | Recovering from a bad step | the retry policy in `loop.py`, not the weights |

    A 256M model cannot emit `(295, 284)` reliably, but it can pick "4" out of five options β€”
    and the harness owns the coordinate lookup, the vocabulary check and the definition of
    success.

    ## Safety, because this thing has your mouse

    - Run it in a **VM, container, or throwaway desktop account**, not your real session.
    - **Dry run is the default**, so you can read a whole episode before anything clicks.
    - Deny lists for typing (`sudo`, `rm -rf`, "password"), widgets (terminal, keychain,
      wallet, bank) and key chords.
    - A **kill-switch file** checked before every action, plus a hard step budget.
    - **Blind grid clicks are refused with real input** unless you pass `--allow-grid`: when
      OCR finds nothing, the fallback index is coordinates with no meaning, and a model
      choosing one is guessing about your desktop.
    - **A brain's "done" is a claim, not evidence**: pass `--verify-text` and the harness
      decides success itself.
    - **Local-first, for real:** pixels and keystrokes never leave the machine.

    ## What works, and what does not

    **Works:** real capture, a real OCR index with coordinates that land on the right pixel,
    a real click that an app reacts to, verified success, the closed vocabulary, the policy
    gates, the no-progress guard, echo detection, the doctor, the CLI, 123 tests β€” and a
    256M VLM that reads a button label from pixels on a clean screenshot.

    **Does not work yet:** reliability on a *live* screen. The 256M VLM echoed the prompt
    there; the 0.5B text planner picks wrong actions; the accessibility-tree detector is not
    implemented, so labels come from OCR only; and the harness still trusts a model's `done`
    when no verifier is given.

    ## Roadmap

    - [x] Capture, OCR index, closed vocabulary, dry-run default, policy gates, doctor
    - [x] A real click on a real window, with a verified outcome
    - [x] Blind-grid-click guard, echo detection, no-progress guard
    - [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing)
    - [ ] Windows run of the live test (workflow included; needs a GitHub mirror)
    - [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows)
    - [ ] Score episodes: replay a trace and count how often the choice was right
    - [ ] marimo console: live screen preview, action log, pause / resume / step

    ## License

    MIT β€” see [LICENSE](LICENSE).
    '''

    mo.md(readme_md)

    return (readme_md,)


@app.cell(hide_code=True)
def repo_files():
    pyproject_toml = '''
    [build-system]
    requires = ["setuptools>=68"]
    build-backend = "setuptools.build_meta"

    [project]
    name = "tiny-computer-agent"
    version = "0.3.0"
    description = "An agentic harness that drives a real computer with a tiny, fully local model - no cloud API."
    readme = "README.md"
    requires-python = ">=3.10"
    license = { text = "MIT" }
    authors = [{ name = "hkblockchaintechnology" }]
    keywords = ["computer-use", "gui-agent", "agentic-harness", "local-first", "tiny-models"]
    classifiers = [
      "Development Status :: 3 - Alpha",
      "Intended Audience :: Developers",
      "Programming Language :: Python :: 3 :: Only",
      "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ]
    dependencies = []   # the harness core imports nothing outside the standard library

    [project.optional-dependencies]
    ocr = ["rapidocr-onnxruntime>=1.3"]        # labels from pixels, no system binary
    desktop = ["mss>=9.0", "pyautogui>=0.9", "pillow>=10"]   # capture + real input
    vision = ["transformers>=4.48", "torch>=2.2", "pillow>=10"]  # tiny model brains
    dev = ["pytest>=8"]

    [project.scripts]
    tiny-computer-agent = "tiny_computer_agent.__main__:main"

    [project.urls]
    Homepage = "https://huggingface.co/hkblockchaintechnology/tiny-computer-agent"

    [tool.setuptools]
    packages = ["tiny_computer_agent"]

    [tool.pytest.ini_options]
    testpaths = ["tests"]
    '''

    license_txt = '''MIT License

    Copyright (c) 2026 hkblockchaintechnology

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all
    copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    SOFTWARE.
    '''

    gitignore_txt = '''__pycache__/
    *.py[cod]
    .venv/
    venv/
    .env
    *.gguf
    models/
    runs/
    traces/
    .ipynb_checkpoints/
    .DS_Store
    '''

    mo.md(f"packaging: {len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars, "
          f"v0.2.0 with ocr/desktop/vision extras")

    return gitignore_txt, license_txt, pyproject_toml


@app.cell
def publish():
    REPO_ID = "hkblockchaintechnology/tiny-computer-agent"

    publish_button = mo.ui.button(
        label="⬆️ Publish README.md to the Hub",
        value=0,
        on_click=lambda value: value + 1,
    )

    publish_button

    return REPO_ID, publish_button


@app.cell
def publish_status(
    REPO_ID,
    gitignore_txt,
    license_txt,
    publish_button,
    pyproject_toml,
    readme_md,
):
    _status = mo.md(
        "This notebook owns the project: the card, the packaging files, the package and "
        "the tests. Click **Publish** to push all of it to the Hub."
    ).callout(kind="info")

    if publish_button.value:
        _token = os.environ.get("HF_TOKEN")
        if not _token:
            _status = mo.md(
                "No `HF_TOKEN` in the environment. Run `export HF_TOKEN=...` "
                "(or `hf auth login`), then restart the kernel."
            ).callout(kind="warn")
        else:
            _api = HfApi(token=_token)
            _root = Path(__file__).parent
            _notebook = Path(__file__)
            _files = {
                "README.md": readme_md,
                "pyproject.toml": pyproject_toml,
                "LICENSE": license_txt,
                ".gitignore": gitignore_txt,
                "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"),
            }
            _folders = ["tiny_computer_agent", "tests"]
            if (_root / ".github").is_dir():
                _folders.append(".github")          # the CI that tests on real Windows
            _missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()]
            if _missing:
                _status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger")
            else:
                try:
                    for _name, _text in _files.items():
                        _api.upload_file(
                            path_or_fileobj=_text.encode("utf-8"),
                            path_in_repo=_name,
                            repo_id=REPO_ID,
                            repo_type="model",
                            commit_message=f"Publish {_name} from marimo",
                        )
                    for _folder in _folders:
                        _api.upload_folder(
                            folder_path=str(_root / _folder),
                            path_in_repo=_folder,
                            repo_id=REPO_ID,
                            repo_type="model",
                            commit_message=f"Publish {_folder}/ from marimo",
                            ignore_patterns=["__pycache__/*", "*.pyc"],
                        )
                    _status = mo.md(
                        f"βœ… Published {len(_files)} files and {len(_folders)} folders to "
                        f"[{REPO_ID}](https://huggingface.co/{REPO_ID})"
                    ).callout(kind="success")
                except Exception as _e:
                    _status = mo.md(f"Publish failed: `{type(_e).__name__}: {_e}`").callout(kind="danger")

    _status

    return


@app.cell
def demo():
    from tiny_computer_agent import AgentLoop, DryRunExecutor, PlannerBrain, Policy
    from tiny_computer_agent.demo import fake_desktop
    from tiny_computer_agent.observe import ImageScreen
    from tiny_computer_agent.perception import IndexedScreen, OcrDetector

    # A real screenshot -> real OCR -> labelled, clickable elements. No display needed:
    # this is the same path --image takes, and it is what makes a tiny model viable.
    _detector = OcrDetector()
    _observation = IndexedScreen(ImageScreen(fake_desktop()), _detector).capture()

    perception_rows = [
        {"label": e.label, "kind": e.kind, "x": e.x, "y": e.y, "w": e.width, "h": e.height}
        for e in _observation.elements
    ]

    perception_episode = AgentLoop(
        screen=IndexedScreen(ImageScreen(fake_desktop()), _detector),
        brain=PlannerBrain(),
        executor=DryRunExecutor(),
        policy=Policy(step_budget=4),
    ).run("click \'Save\' to save the document")

    _clicked = next((s.result.note for s in perception_episode.steps if s.action and s.action.kind == "click"), "n/a")

    mo.vstack(
        [
            mo.md(
                f"**Perception, from pixels.** OCR read **{len(perception_rows)} labels** off a "
                f"synthetic screenshot; the planner then chose one and the harness resolved it to a real pixel."
            ),
            mo.ui.table(perception_rows, selection=None),
            mo.md(f"`{_clicked}`"),
            mo.md(f"`{perception_episode.render().splitlines()[1]}`"),
        ]
    )

    return


if __name__ == "__main__":
    app.run()