import marimo __generated_with = "0.25.2" app = marimo.App(width="medium", auto_download=["html"]) with app.setup(hide_code=True): # Everything imported once, for every other cell of this notebook. # `materialize` copies only the imports each module actually uses into # /marimo/omni/.py, and completes its pool from NAME_IMPORTS. from __future__ import annotations import marimo as mo from PIL import Image from PIL import Image, ImageDraw, ImageFilter from PIL import ImageDraw from PIL import ImageFilter from dataclasses import asdict from dataclasses import asdict, dataclass, field from dataclasses import asdict, dataclass, replace from dataclasses import dataclass from dataclasses import field from dataclasses import fields from dataclasses import replace from pathlib import Path from torch import nn from typing import Any from typing import Mapping import argparse import atexit import gzip import hashlib import importlib import io import json import logging import math import numpy as np import os import queue import random import re import shutil import signal import subprocess import sys import tempfile import threading import time import torch import torch.nn.functional as F @app.cell def _(): return @app.cell(hide_code=True) def handoff(): # --- HANDOFF: read this first if you have no memory --------------------------- # A fresh session starts with no history, and the box dies without warning. Only # *this notebook* is snapshotted -- everything else under /marimo is gone. So this # cell is the memory. Keep it SHORT, keep it TRUE, and curate it: a memory that # contradicts itself is worse than no memory. Update it the moment something changes. mo.md(r""" # Where we are — read this first ## The thirty-second version * The box relaunches on a schedule — **HKT 03:00 = 19:00 UTC** — and erases everything under `/marimo` except **this notebook**. There is no grace period. * **To resume:** the `bootstrap` cell above has already restored the toolchain, so run `restore_box.py` → `swarm.sh` → `backup_daemon.py` → `watch.py` → `final_flush.py`, then `setup` → `config` → `materialize` in the notebook. `restore_box.py` **before** any training. * **The Hub is the archive** — checkpoints, *every* milestone ever pushed, the ops toolchain, the card assets, the face sprites. Local pruning deletes local copies; the Hub keeps them. * **Do not trust `latest`.** What serves is chosen by two `active.json` files, and the measured-best speech checkpoint is step **39 002** (47% exact match vs 35.5% for `latest`). * **Three rules that each cost real work:** never run `backup_now.py` by hand (the commit budget is shared with the trainers); never push binary media with `create_commit` (use git-lfs and verify by hash); never retry the seeding trick to warm-start a run. * The face is now a **photoreal sprite puppet** whose ten mouth shapes are selected by the model's own phoneme timeline. The card describes it honestly. The rest is detail, and every claim in it was verified. A fresh session has no memory of any of this, and the box can die without warning. **Only `notebook.py` is snapshotted** — `/marimo/omni`, `/marimo/runs`, `/marimo/data` and `/marimo/logs` are ephemeral. So knowledge belongs in a *cell* (this one) or in the Hub, and so does every code change: a file written to `/marimo/omni/*.py` is overwritten by the next `materialize`. If you are reading this and something below is out of date, **fix it here, now** — do not batch it up, because there is no quiet moment later. ## If the user says "resume" The box relaunches on a schedule (**HKT 03:00 = 19:00 UTC**) and erases everything but the notebook. In this order: ```bash # 0. the `bootstrap` cell above has already run: it restored the toolchain from the Hub python restore_box.py # shared state, corpora, newest run per stage (~5 min) bash swarm.sh # resume every stage from its own checkpoint nohup python backup_daemon.py --interval 1200 --code-every 3 & # save every 20 min nohup python watch.py & # liveness, every minute nohup python final_flush.py & # armed for the next 18:48Z: saves the last minutes ``` then in the notebook `setup -> config -> materialize -> monitor`, and check `python omni.cli stages` (hub step vs local step) and `python check_served.py`. **`restore_box.py` must run before any training starts**: the vocabulary cache decides every run id, and training begun without it invents new runs and looks like a fresh start. ## Scope This box only. Four sibling accounts (`bluemorholimited`, `pinkelephantlimited`, `dragonlimited`, `professionalserviceshk`) hit the same class of failures but **their jobs are finished — leave them alone.** This token cannot write to them: it is scoped to `hkblockchaintechnology` alone. `box_doctor.py` and `docs/RUNBOOK.md` exist for *this* box. ## What this project is OmniVLM: one transformer trunk trained **from scratch**, whose *heads* are the modalities — eyes, text, phonemes, durations, mel, waveform, and a rendered talking face. `image + question -> answer -> speech + lips`, one model, no pretrained parts. | repo (both public) | holds | |---|---| | `hkblockchaintechnology/omni-vlm-ckpt` | checkpoints, optimizers, metrics, alignments, decoded corpus, ops scripts | | `hkblockchaintechnology/VLM-all-inclusive` | `notebook.py`, `omni/*.py`, tests, scripts, model card, `assets/`, samples | Token `/marimo/.hf_token`. `hub.CKPT_PRIVATE = False`. ## Orient in 5 minutes ```bash python omni.cli stages # per stage: hub step, local step, run id python check_served.py # which checkpoint inference actually serves tail -3 /marimo/logs/watch.jsonl # liveness: is every stage advancing? cat /marimo/logs/backup_daemon.json # last backup cycle, what it pushed, errors python box_doctor.py --repo hkblockchaintechnology/omni-vlm-ckpt --no-probe ``` ## State — 2026-11-17 17:55 | stage | step | of plan | metric | note | |---|---|---|---|---| | **vlm** (the only real pretraining) | 12 000 | **150%** | loss 0.062 | **done**, converged, not running | | ear | 251 001 | 63% | ctc **0.0067** | running; also feeds the alignments | | speech (v2) | 81 503 | 54% | mel dominates | running; **serves** inference at step 39 002 | | vocoder | 503 001 | 72% | stft **0.866** | running | | captions | 26 004 | **5%** | loss ~0.5 | running; the plan is unreachable, see below | | align | 13 100 clips | — | ear ctc 0.014 | complete | **299 distinct checkpoints** archived on the Hub across 5 stages, from **873 509 trained steps** of a 1 758 000-step plan (50%). A checkpoint is written every 500-6 000 steps depending on the stage; `latest/` is overwritten each time, so milestones are the durable record, and the Hub keeps every milestone ever pushed while local pruning keeps 3. **The captions plan is not reachable and should be re-thought, not fed more hours.** At its measured 1.2 s/step, the remaining 474 000 steps are ~7 days of GPU. The other stages are hours: speech ~8 h, ear ~14 h, vocoder ~4 h. Four trainers + `watch.py`, `backup_daemon.py` and a deadline-armed `final_flush.py` are up. **The commit ledger is over its allowance (102/100)** — every `create_commit` is deferred, so publish with `publish_all_lfs.sh` (a real `git push`, which does not consult the ledger). ## Traps that have already cost real work 1. **A run id is not a way to find a run.** It hashes the *serialised* config, and `RunConfig` writes 6 geometry keys while `VLMConfig.to_dict()` writes 14 — so the same model hashes two ways and one launcher names a run the other never made. That looked like "no checkpoint" and silently trained from zero. Use `find_resumable_run`, never a recomputed id. 2. **`materialize` reads `notebook.py` from disk.** If the editor has not saved, it regenerates *stale* modules — how the real-data v2 stage was lost once. `_source_stamp` now raises instead; if it raises, run the cell again. 3. **Editing `omni/*.py` on disk is invisible.** The next `materialize` overwrites it. Every change goes into the owning `lib_*` cell. 4. **Never raise `--steps` to extend a run.** The LR is a cosine over that number, so raising it *re-heats* the rate and knocks a converged model off its minimum (0.82 -> 1.14 once). `store.schedule_total` pins the horizon to the run. 5. **A flat metric is usually `lr == 0`, not convergence.** The vocoder sat at `stft` 0.9130 for 30 000 steps with `lr = 0.000e+00`: the cosine had passed its horizon, so every gradient was multiplied by zero. `lr_warmup_cosine` now floors the tail at 5%. Check `base_lr * cosine` before believing a plateau. 6. **Pruning deletes milestones.** `keep_milestones` destroyed the vocoder's best weights (0.82) in favour of newer, worse ones. `prune_milestones` now protects whatever `active.json` names — so **pin with `set_active` before experimenting**. 7. **`latest` is not `best`.** Measured, same 200 questions, only the checkpoint differing: `speech-xl-v2-f93271` @39002 = **47.0%**, @49502 (`latest`) = **35.5%**. More training made question-answering *worse* (the speech stage fine-tunes the whole trunk on a speech objective). Choose serving by **measurement**, then pin. 8. **`BEST_METRIC` names must match the logged keys.** It says `"ctc"`/`"stft"` while the stages log `"ctc_loss"`/`"stft_loss"`, so best-checkpoint protection silently never fired for the ear or the vocoder. 9. **A silent fallback hides a broken loader.** `OmniSystem.load` logged "loaded speech trunk step 6502" for hours while the better model was unreachable, because something loadable was always found instead. When code has a fallback, log **what it fell back *from***. 10. **Never push `state.json` without its payload.** The pointer is written last so its presence implies the weights arrived. 11. **A private repo has a storage ceiling.** This one hit it at ~65 GB: every push failed with `400 Private storage limit reached` **while training continued**. Hence public. 12. **The hourly commit budget is shared and finite (~100-128/hour).** Three ways it bites: * a busy stage starves the others — the vocoder spent 33/hour because `ckpt_steps=500` against a stage at 0.064 s/step, i.e. a checkpoint every 32 s. *Cadence is part of the job spec*; it now checkpoints every 3 000-6 000 steps / 30 min; * **never run `backup_now.py` by hand in a loop** — one session spent ~20 of 100 commits on `code:`/`ops:` and every caller was then deferred, including the trainers' pushes. **It happened for real the same day: the ledger read 103/100, `allowed=False`, and the speech run fell 7 500 steps behind the Hub.** The cure is one *gated, priority* flush — `hub.resync_local_runs(..., priority=True)` — which bypasses the allowance and pushes only what is ahead. Use it whenever the exposure window matters, not only at the deadline; * a retry can duplicate a commit that already **landed** (a response timeout looks like a failure). `_already_committed` now checks before retrying. `final_flush.py` passes `priority=True` — the only thing that should bypass the allowance. 13. **A batched backup commit is a single point of failure.** One request carrying ~15 GB was refused, and then *nothing* was saved while only a log line said so. It now falls back to one commit per run. Any "one request does everything" needs a fallback. 14. **`swarm.sh --stop` did not stop everything**, and the relaunch added a second copy of each stage — nine trainers, four duplicated onto the same run directories. Whatever writes a run dir must be singular. **Count the processes after a stop, before starting anything.** 15. **`upload_file` / `create_commit` can report success while uploading NOTHING.** On `huggingface_hub` 1.24 storage goes through **Xet**: three commits claimed the 1.54 GB bundle, returned a commit URL in ~0-2 s, and the tree never changed. **A commit URL is not evidence.** Verify `lfs.sha256`/`size` via `api.list_repo_tree`, or use `git-lfs`, which fails loudly. (`git-lfs` is installed at `/usr/local/bin/git-lfs` but **is erased by a relaunch** — reinstall from the GitHub release if a large upload is needed.) 16. **The model-card template is a raw string that is `.format()`-ed.** Double any literal brace; a backslash stays a backslash, so write `torch[.]hub` rather than `torch\.hub`. Render the card before trusting it. 17. **Do not let a dashboard spend the API quota.** `hub_status` is cached; the monitor used to hit the Hub every 5 s, one request per run. ## Serving: four stacked faults, now fixed (this was the headline) The *worse* model was in front of everyone and nothing said so: `shared/active.json` was hours stale; `meta = ck["payload"].get("meta", {})` always returned `{}` (a state_dict has no meta); the vocabulary was always built as v1, so the v2 trunk could not be loaded at all; and `BEST_METRIC` never fired (trap 8). Now the vocabulary is chosen by matching `len(CombinedVocab)` against the checkpoint's own `tok.weight` row count — the embedding cannot disagree with the weights — and **18.5% became 47.0%**. ## The face, and the art direction The talking head was an abstract diagram; it is now a **stylised character** — soft-shaded skin, modelled lips with gloss, real eyes with catchlights and lashes, hair with a rim light. Deliberately *tasteful and appealing* rather than sexualised: this is the surface a person sees from an agent, a phone or a robot. **Do not undo that choice.** Still driven only by the four mouth parameters, so the model's own phoneme timeline moves every frame — the renderer got prettier, the model did not change. ~358 ms/frame at 384 px (~86 s per 240-frame movie). Two hard-won art lessons: hair was invisible until the backdrop was made luminous (dark on dark reads as a hole), and a solid hair mass erased the neck (side locks fixed it). ## Binary media: use git-lfs, and verify by hash `create_commit` put 10 files into one commit for the samples. **The four `.wav` files landed; the four `.mp4` files did not** — the tree kept the previous revision, at a *different* size, while the commit returned a URL. Same commit, same kind of file (both are LFS), different outcome. Retrying changed nothing. What worked: **git-lfs** (`/usr/local/bin/git-lfs`, 3.5.1 — erased by a relaunch): ```bash export GIT_LFS_SKIP_SMUDGE=1 git clone --quiet https://user:$TOKEN@huggingface.co/hkblockchaintechnology/VLM-all-inclusive /tmp/pub_lfs cp /marimo/out/*.mp4 /marimo/out/*.wav /marimo/out/demo_report.json /tmp/pub_lfs/samples/ cd /tmp/pub_lfs && git add -A samples && git lfs status # names every object and its sha git commit -m ... && git push # prints the transfer, or fails ``` `git lfs status` listed all four objects with their old→new shas, the push reported `Uploading LFS objects: 100% (4/4)`, and afterwards every one of the eight files was confirmed by downloading it and comparing `sha256`. **A commit URL is not evidence of an upload; a hash comparison is.** ## I tried the adversarial-vocoder fix and it went wrong — read before retrying Attempted: warm-start a *separate* run (`vocoder-xl-gan-bfba0a`) from the pinned milestone `mile-00355502`, launch it with `--set adversarial=true`, leave the working run alone. **What actually happened.** `find_resumable_run` ranks candidates by `updated_at` on the Hub. I seeded the new run's directory locally and published it once at 14:31 — but by 15:09 the *main* vocoder run had pushed a newer timestamp, because it is the one still training. So discovery chose `vocoder-xl-main-bfba0a` and launched **that** with `adversarial=true`: the run I was trying to protect got the experimental objective instead. Caught within ~500 steps (lr was at the 5% floor, so the damage is negligible), killed, and the stage restarted from the swarm with `adversarial=false`; it is healthy at `stft` ~0.873, its best yet. **The lesson: discovery is not a warm-start mechanism.** Seeding a run directory and hoping `updated_at` wins is fragile *by construction*, because the run you are trying to leave alone is exactly the one that keeps updating. A real warm start needs an explicit parameter — a `base_run`/`base_stage` on the vocoder stage, mirroring the `base_stage` that `speech` already has. **Do not retry the seeding trick.** **Two things this episode did buy.** (1) The guard added to `train.execute()` — refusing a stage whose `start >= steps` — fired immediately and turned "ran and exited in 4 s" into `nothing to do: run ... is at step 400001 but --steps is 400000`. Before it, a stage whose budget was exhausted exited **rc=1 / rc=0 in seconds having done nothing**, which reads as success. (2) `swarm.sh --only ` made it possible to restart one worker without starting duplicates of the others. `vocoder-xl-gan-bfba0a` still exists, locally and on the Hub, holding the pinned weights and re-labelled with an adversarial objective it never actually trained under. Use it properly with a warm-start parameter, or delete it. ## The evidence plan — what to build next, ordered by how many challenges it closes Derived from the 100-challenge bank (`docs/CHALLENGES.md`, also in the card's Q&A). This is the training design that the challenges dictate: 1. **Train the phone/duration/mel heads with scheduled sampling** (feed their own predictions), or add an autoregressive loss, and predict EOS from a real stop token. Then re-run `validate_voice.py` until WER < 0.4. *Closes: the voice is gibberish, exposure bias, the samples are worthless, the headline claim fails.* 2. **Freeze the trunk during the speech stage** (or drop its LR for the trunk). Re-measure VQA at every checkpoint and stop when it falls. *Closes: more training made answering worse.* 3. **Put the independent WER test in the eval loop with a hard gate** — it is the only test here that can falsify anything. *Closes: self-referential evaluation.* 4. **Confidence intervals** on exact-match (bootstrap the 200 questions) and 3 seeds. *Closes: too few samples, no variance.* 5. **A random-init and a majority-class baseline** on the same 200 questions. *Closes: no baseline.* 6. **A human rating sheet** — even 5 raters on fixed seeds — for the face and the audio. *Closes: judged by a model, not a person.* **The ratio is the point.** Of 100 rehearsed attacks: ~11 answered with a measurement I would defend, ~30 with a measured number that is *bad* and published anyway, ~25 open admissions. **The engineering claims are hard (resume, checkpointing, rebuild, from-scratch); the modelling claims are soft.** The two most damaging facts — the voice does not work, and more training makes answering worse — were invisible to every metric the project had. ## THE VOICE DOES NOT WORK — measured, and the loss could not see it `validate_voice.py` sends text through the model's own voice path and hands the audio to **Whisper**, which has never seen this project, its data or its vocabulary. Result over five sentences: **mean WER 0.966**, with each "sentence" lasting **15 s where 2 s is expected** — the decoder loops instead of emitting EOS. Meanwhile the training metrics were excellent: `phone_acc 1.000`, `dur_loss 0.005`, falling `mel_loss`. **Why they could not see it:** `forward_speech` is **teacher-forced** — it feeds the *true* previous phone (`phone_in = shift(phone_ids)`), so the head is never trained on its own output. Autoregressive `synth()` then feeds its own predictions and drifts. This is exposure bias, and the fix is scheduled sampling (or an autoregressive loss) during training. **The general rule this establishes: a teacher-forced loss is not evidence about generation.** Any stage that will be *used* autoregressively must be evaluated autoregressively, by something independent. Run `validate_voice.py` after every speech change — it is the only test in this project that can falsify the voice claim, and it did. ## The served checkpoint was deleted by pruning — recovered, and fixed `shared/active.json` named `speech-xl-v2-f93271` at step **39 002**, the checkpoint measured at **47.0%** exact match. But serving reads the *run-local* `active.json`, which said **51 002** (the same checkpoint measured **35.5%** — more training made answering *worse*). There are two "active" files with different readers: | file | written by | read by | means | |---|---|---|---| | `runs//active.json` | `set_active`, on a new best *training* metric | `load_for_inference` | **the served step** | | `runs/shared/active.json` | `select_served.py` | `open_stage_run` | which **run** serves a stage | `prune_milestones` protected only the first, so it deleted `mile-00039002`. It was recoverable **only because the Hub archives every milestone ever pushed** — local pruning deletes local copies, the Hub keeps accumulating. `prune_milestones` now protects the **union** of both pins, and the recovered milestone is back: serving is step 39 002 again. **Two follow-ups:** (1) the serving decision should be *measurement*-based, not training-metric-based — `select_served.py` has the right idea but implements it at run level, so it cannot express "39 002, not the latest"; (2) the published bundle, the published samples and the served checkpoint are three different states. A stranger experiences the **bundle**, so refreshing it with the measured-best trunk is the highest-value publish left. ## The face is now photoreal — sprite-driven, and why it changed The drawn face was readable but a vision model kept calling it *"cartoon-like, exaggerated, amateurish"*, and that matched what a human said about the card. It is now **photoreal**: * one beauty portrait from SDXL (`sdxl_gen.py`, `SG161222/RealVisXL_V4.0`), plus * **ten mouth states**, one per viseme, each produced by **img2img from that same base at a fixed seed** and composited through a **feathered elliptical mask** (`sdxl_mouths.py`), so the identity is **pixel-identical outside the mouth** (measured: mean |diff| 0.01 there) and cannot drift between shapes. * `render_frame` picks the three nearest visemes in mouth-parameter space (each axis scaled by its range) and cross-fades them. **The model's own phoneme timeline still drives the lips** — it selects the shapes instead of moving a drawn polygon. Verdict, same oracle, same question: *"cartoon-like, lack of detail, amateurish"* became *"a high level of detail and realism, making it visually appealing"*. It also renders **2x faster** (165 ms/frame vs 358), so a movie went from ~150-250 s to **27 s**. **The bug worth remembering: CLIP truncates at 77 tokens, and it truncates from the END.** My prompt put a long style prefix first and the mouth instruction last, so the only words that described the mouth were being dropped — the ten shapes differed by a mean of 4 on a 255 scale, and only 4% of the mouth pixels changed. Putting the shape first and shortening the prefix took that to 44%. The warning *was* in the log ("truncated because CLIP can only handle sequences up to 77 tokens") and was easy to read past. **Durability.** The sprites are published at `assets/face/mouth_.png` and fetched on demand by `avatar._sprite_file`; if they are missing entirely, `render_frame` falls back to the drawn face rather than failing. They must go **through git-lfs**: a 1.5 MB PNG added as a plain git object is refused by the Hub's pre-receive hook, and `git lfs status` is what tells you which you have (`Git:` vs `LFS:`). **Honest limits:** ten discrete mouth shapes, no head turn, and **no blinking** (`blink` is ignored by the sprite path). It is a puppet with the model's timing, not a learned talking-head model. The card says so. ## The test suite is a real safety net — run it after touching the avatar `PYTHONPATH=/marimo python -m pytest -q /marimo/repo/tests` — **71 tests, all green** as of 14:29. It caught one of my own regressions, and the way it caught it is worth keeping: `test_render_frame_is_deterministic_and_mouth_drives_pixels` counted dark pixels across the *whole frame* as a proxy for "the mouth opened". Once the background gained shading that proxy was mostly measuring the backdrop, and it stopped separating open from closed. The fix was to measure the mouth region and compare **brightness** rather than counting pixels under a fixed threshold — at test resolution the opening is only a few pixels across and is smoothed into its surroundings, so a threshold count finds *nothing*. Same claim, measurable. (A separate `test_captions` bug — `train[-1] == train[len(train)-1]` comparing tuples of arrays — was pre-existing.) Measuring that test also exposed a real rendering flaw: at high `open` the teeth and tongue covered almost the entire opening, so a wide-open mouth showed no dark interior at all. The teeth are now 23% of the gap instead of 34%, and the tongue is smaller and sits lower. ## Eyes and taste: two new tools, and what they are actually good for **`critique.py`** (project env) loads `Qwen/Qwen2.5-VL-3B-Instruct` and asks it to describe and judge an image. **`sdxl_gen.py`** (in `/marimo/venv-sd`, see below) generates beauty references with `SG161222/RealVisXL_V4.0` — 1024 px in ~7 s. What the oracle is worth, established by experiment rather than assumption: * it is **deterministic** — the same image twice gives byte-identical text (only the "loaded in 4s/5s" line differs). So a changed verdict means a changed *image*, and its advice can be compared across versions; * its aesthetic bar is **photorealism**. On a real photograph: *"a high level of detail and realism, making it visually appealing"*. On our deliberately stylised character: *"exaggerated features... amateurish"*. So its "too exaggerated" complaint is a stable statement that the character is stylised — a choice, not a defect; * therefore it is a good critic for **facts** (flat shading, missing hair strands, wrong proportions, invisible features) and a poor oracle for **taste** on stylised work. It earned its keep on the facts. Its critique drove the face from *"somewhat amateurish and unintentional"* to *"well-executed... polished and professional"* by fixing eye width (the eyes were **twice** the correct size — an eye is about a fifth of the face width), adding four-part form shading, hair strands, and a parting. **Then it was frozen**: further tuning against a photorealism-biased oracle is chasing noise, and the samples and figures must agree with one another. ## Environment: keep diffusion out of the project venv `diffusers` requires `huggingface-hub>=1.5`, and `transformers<5` requires `<1.5` — they cannot coexist. Installing `diffusers` into the project environment **upgraded `huggingface-hub` to 2.4.0**, which `transformers 5.14.1`, `tokenizers` and `datasets` all reject; the fix was to pin it back to **1.24.0** immediately (`pip install 'huggingface-hub==1.24.0'`). New processes — including every trainer the swarm restarts — import that hub, so this was a live risk to the save path. The working arrangement is a **fully isolated venv** (`uv venv /marimo/venv-sd`, no `--system-site-packages`): nothing is shared, so pip resolves `diffusers 0.43` + `transformers 5.23` + `hub 2.4` freely. A venv *with* `--system-site-packages` does **not** work — it inherits the project's pins and resolution becomes impossible. **`/marimo/venv-sd` and the 4 SDXL references in `/marimo/sdxl/` are erased by a relaunch.** Rebuild with `uv venv /marimo/venv-sd && uv pip install --python /marimo/venv-sd/bin/python diffusers transformers accelerate safetensors`. ## What actually limits the quality now `phone_loss 0.0002`, `phone_acc 1.000`, `dur_loss 0.005` — the *timing* is saturated. The remaining loss is `mel_loss` (~0.76), and the vocoder that turns mel into sound is the weak link in what a listener hears. **The work belongs on the vocoder and the mel decoder, not on more trunk steps.** The vocoder is at `stft` 0.885 and, since the LR floor, moving again. ## Done — do not redo * from-scratch claim **verified**: zero `from_pretrained` / `torch.hub` / `transformers` / `timm` / `whisper` / `wav2vec` / `hifigan` across all 20 source files. Only downloads are LJSpeech, flickr30k and CMUdict (a *lexicon*, not a model). Scenes are generated. * re-align with the 130k ear: 13 100 clips, `len(dur) == len(phones)`, `sum(dur)/n_mel == 1.000`, and **`speech` is already running on them** — do not restart it for that reason again. * model card with 6 generated figures and a falsifiable "from scratch" section. * tooling: `restore_box.py`, `swarm.sh --only`, `smoke.sh`, `watch.py`, `backup_now.py`, `backup_daemon.py`, `final_flush.py`, `check_served.py`, `select_served.py`, `box_doctor.py`. * the fixed serving path, the LR floor, the pinned vocoder milestone, the curated memory. ## Open work, in value order 1. **The vocoder.** Plateaued at 0.885. The adversarial term is implemented (`MultiResolutionDiscriminator` in `speech.py`) and `adversarial=false`; a `vocoder-large-gan` run exists. The project's own note says a pure STFT loss gives *flat* audio, and flat is what the samples sound like. Warm-start it from the pinned milestone. 2. **The speech stage is eroding the trunk's question-answering** (trap 7). Freeze the trunk during speech training, or lower the fine-tuning rate — a design change, not a tweak. 3. **`weights/omni_weights.pt` and `samples/` lag the checkpoints**, and large uploads need git-lfs (trap 15). Refresh with `publish_two_phase.py` before claiming anything publicly. 4. **`lips_vs_audio` is weak** (-0.26 to +0.08 across samples). Worth measuring properly rather than asserting sync. """) return @app.cell(hide_code=True) def bootstrap(): # --- bootstrap: put the ops toolchain back ----------------------------------- # The box relaunches (scheduled: HKT 03:00) and *everything* under /marimo except # this notebook is erased -- including restore_box.py, swarm.sh, watch.py and # backup_daemon.py. So the notebook has to be able to rebuild its own tooling, or # a fresh session has no way to fetch the tool that fetches things. # # Run this cell first after a relaunch. It only needs the token and # huggingface_hub, and it never overwrites a file that is already here, so a local # edit can never be clobbered by the copy on the Hub. CKPT_REPO = "hkblockchaintechnology/omni-vlm-ckpt" PUBLIC_REPO = "hkblockchaintechnology/VLM-all-inclusive" def bootstrap_ops(dest="/marimo", log=print, force=False): """Restore ops/* (scripts, launchers, daemons) from the checkpoint repo. Returns {name: "restored" | "present" | error}. `force=True` overwrites. """ from huggingface_hub import HfApi, hf_hub_download import shutil as _sh tok = os.environ.get("HF_TOKEN") or ( Path("/marimo/.hf_token").read_text().strip() if Path("/marimo/.hf_token").exists() else "") if not tok: log("no HF token at /marimo/.hf_token - cannot bootstrap") return {} os.environ["HF_TOKEN"] = tok os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1") try: paths = [p for p in HfApi(token=tok).list_repo_files(CKPT_REPO, repo_type="model") if p.startswith("ops/")] except Exception as exc: log(f"cannot list {CKPT_REPO}: {type(exc).__name__} {exc}") return {} Path(dest).mkdir(parents=True, exist_ok=True) # a missing dest used to raise per file out = {} for p in sorted(paths): name = Path(p).name target = Path(dest) / name try: if target.exists() and not force: out[name] = "present" continue _sh.copyfile(hf_hub_download(CKPT_REPO, p, token=tok), target) if name.endswith(".sh"): target.chmod(0o755) out[name] = "restored" except Exception as exc: out[name] = f"{type(exc).__name__}: {exc}" got = [k for k, v in out.items() if v == "restored"] present = [k for k, v in out.items() if v == "present"] failed = {k: v for k, v in out.items() if v not in ("restored", "present")} log(f"toolchain: {len(got)} restored, {len(present)} already here") if failed: # Naming the failures matters: the first version printed "0 restored, 0 already # here" when every file had errored, which reads exactly like success. A # bootstrap that cannot bootstrap must say so. log(f" {len(failed)} FAILED:") for k, v in list(failed.items())[:6]: log(f" {k}: {v}") return out def next_steps(log=print): """Print the cold-start sequence, so it is not re-derived every time.""" here = {p.name for p in Path("/marimo").glob("*")} need = [s for s in ("restore_box.py", "swarm.sh", "watch.py", "backup_daemon.py") if s not in here] log("") log("== cold start ==") if need: log(f" MISSING: {', '.join(need)} - run bootstrap_ops() first") log(" 1. python restore_box.py # shared state, corpora, newest run per stage (~5 min)") log(" 2. bash swarm.sh # resume every stage at once") log(" 3. nohup python backup_daemon.py & # save to the Hub every 20 min") log(" 4. nohup python watch.py & # one liveness line a minute") log(" 5. in this notebook: setup -> config -> materialize -> monitor") log(" 6. python omni.cli stages # compare hub step vs local step") return need _boot = bootstrap_ops() _boot_need = next_steps() return @app.cell(hide_code=True) def config(): # --- Configuration. Everything that identifies a run lives here. ------------- CFG = dict( preset="xl", # nano / small / base / large / xl / xxl steps=8000, # VLM stage budget (resumable, so this can change) batch=384, accum=1, lr=4e-4, log_every=20, ckpt_steps=250, # checkpoint + push to the Hub every N steps ckpt_seconds=900, # ... or every N seconds, whichever comes first data_seed=0, img_size=128, run_tag="main", hf_repo="hkblockchaintechnology/omni-vlm-ckpt", # public: checkpoints (weights, optim, metrics) public_repo="hkblockchaintechnology/VLM-all-inclusive", # public: code + card ) RUN_ROOT, LOG_DIR, OUT_DIR = "/marimo/runs", "/marimo/logs", "/marimo/out" for _p in (RUN_ROOT, LOG_DIR, OUT_DIR, RUN_ROOT + "/shared", "/marimo/omni"): Path(_p).mkdir(parents=True, exist_ok=True) # Token: env var first, then a sandbox-local file (never written into the repo). _env_tok = os.environ.get("HF_TOKEN", "") HF_TOKEN = _env_tok or (Path("/marimo/.hf_token").read_text().strip() if Path("/marimo/.hf_token").exists() else "") if HF_TOKEN: os.environ["HF_TOKEN"] = HF_TOKEN print("token:", "found" if HF_TOKEN else "MISSING - put it in /marimo/.hf_token", "| GPU:", torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu") return CFG, HF_TOKEN, LOG_DIR, OUT_DIR, RUN_ROOT @app.cell(hide_code=True) def lib_hub(): # ===== omni/hub.py ===== module body # Imports live in `setup`; `materialize` writes this back out to # /marimo/omni/hub.py so detached trainers and future sandboxes import # exactly the code that lives in this notebook. #: The checkpoint repo is *public*. A private repo is capped by the account's #: private-storage allowance -- this one hit "private storage limit reached" at #: ~65 GB and every checkpoint push started failing with a 400, which is the #: worst possible moment to discover a ceiling. Public storage has no such cap, #: and nothing secret is in there: model and optimizer state, metrics, logs, the #: decoded corpus index, and the ops scripts that produced them. The token is #: only ever read from $HF_TOKEN or /marimo/.hf_token, never written into a repo. #: The Hub *ignores* `private` for a repo that already exists, so this decides #: what happens only if the repo is ever recreated. CKPT_PRIVATE = False def canonical(obj): return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) def config_hash(cfg: dict) -> str: return hashlib.sha256(canonical(cfg).encode()).hexdigest()[:16] def atomic_write_bytes(path, data: bytes): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name + ".", suffix=".tmp") with os.fdopen(fd, "wb") as f: f.write(data) f.flush() os.fsync(f.fileno()) os.replace(tmp, path) def atomic_write_json(path, obj): atomic_write_bytes(path, json.dumps(obj, indent=2, default=str).encode()) def atomic_torch_save(path, obj): buf = io.BytesIO() torch.save(obj, buf) atomic_write_bytes(path, buf.getvalue()) def rng_state() -> dict: import random import numpy as np s = { "python": random.getstate(), "numpy": np.random.get_state(), "torch": torch.get_rng_state(), } if torch.cuda.is_available(): s["cuda"] = torch.cuda.get_rng_state_all() return s def set_rng_state(s: dict): import random import numpy as np random.setstate(s["python"]) np.random.set_state(s["numpy"]) torch.set_rng_state(s["torch"].cpu() if isinstance(s["torch"], torch.Tensor) else s["torch"]) if "cuda" in s and torch.cuda.is_available(): torch.cuda.set_rng_state_all(s["cuda"]) def run_identity(cfg: dict) -> dict: """The part of a config that makes a run a *different* run.""" from omni.config import IDENTITY_KEYS as _KEYS # single source of truth return {k: cfg[k] for k in _KEYS if k in cfg} def stage_run_id(stage: str, cfg: dict) -> str: """Canonical run id, computed identically by every process that touches a stage, so a fresh sandbox finds the checkpoint a detached trainer pushed.""" from omni.config import IDENTITY_KEYS as _KEYS # single source of truth assert set(run_identity(cfg)) <= set(_KEYS) ident = {**run_identity(cfg), "stage": stage} tag = cfg.get("run_tag") or "main" return f"{stage}-{cfg.get('preset', 'xl')}-{tag}-{config_hash(ident)[:6]}" def find_latest_stage_run(repo_id: str, stage: str, token=None): """Newest `runs/-*/meta/state.json` in a repo. Discovery beats recomputing a run id: the ear and the vocoder are trained with their own hyper-parameters, and this keeps every consumer (inference, the aligner, the notebook) pointed at whatever is actually newest. """ repo = RemoteRepo(repo_id, token=token) if not repo.enabled: return None try: files = repo.api().list_repo_files(repo_id) except Exception: return None best = None for f in files: parts = f.split("/") if (len(parts) == 4 and parts[0] == "runs" and parts[1].split("-")[0] == stage and parts[2] == "meta" and parts[3] == "state.json"): st = repo.read_json(f) if st is None: continue if best is None or st.get("updated_at", 0) > best[1].get("updated_at", 0): best = (parts[1], st) return best #: Config keys that decide whether a checkpoint can be resumed *into* a run. #: Deliberately narrow: the vocabulary is checked against the checkpoint's own #: `meta.pt` instead (see `checkpoint_n_words`), because a *recorded* config can #: disagree with the weights it describes. RESUME_COMPAT_KEYS = ("preset", "img_size") def config_matches(resume_cfg: dict, ident: dict) -> bool: """True when a stored run's config is compatible with the one we want to run. Only the keys in `RESUME_COMPAT_KEYS` are compared, and only those present on both sides, so a config that predates a new knob does not look incompatible. """ for k in RESUME_COMPAT_KEYS: if k in ident and k in resume_cfg and resume_cfg[k] != ident[k]: return False return True def checkpoint_n_words(repo, run_id: str): """The embedding-table size a checkpoint *actually* has, from its own meta. `latest/meta.pt` is written from the model object, so it cannot disagree with the weights. The recorded config can: the real-data run was launched by a path that stamped the vision-language vocabulary (8 011 words) into its config while the model itself was built on the real-data vocabulary (16 380 words). Trusting the config there would load an 8 011-row embedding into a 16 387-row table, or -- worse -- refuse to resume and quietly start from zero. Returns None for a run whose meta does not carry a token count (`ear` and `vocoder` do not, and they have no embedding table to get wrong). """ meta = repo.read_json(f"runs/{run_id}/latest/meta.pt") if isinstance(meta, dict) and meta.get("n_words"): return int(meta["n_words"]) return None def find_resumable_run(repo_id: str, stage: str, ident: dict, token=None, log=None, n_words=None): """Newest run of `stage` that `ident` can legally resume -> (run_id, state). `n_words` is the total size of the embedding table this launch will build (words + the special tokens the combined vocabulary prefixes); when given, it is checked against the checkpoint's own meta. A run id is a hash of the *serialised* config, so it depends on how that config was assembled: `RunConfig.PRESETS` writes six geometry keys, `VLMConfig.to_dict()` writes fourteen. The same model therefore hashes two ways, and an id computed by one launcher can name a run the other never created -- which looked exactly like "no checkpoint yet" and silently restarted training from zero. So ask the Hub what is actually there. """ repo = RemoteRepo(repo_id, token=token) if not repo.enabled: return None, None try: files = repo.api().list_repo_files(repo_id) except Exception: return None, None best, rejected = None, [] for f in files: parts = f.split("/") if not (len(parts) == 4 and parts[0] == "runs" and parts[2] == "meta" and parts[3] == "state.json" and parts[1].split("-")[0] == stage): continue st = repo.read_json(f) if not st or not st.get("step"): continue rid, step = parts[1], st.get("step") if not config_matches(st.get("config") or {}, ident): rejected.append(f"{rid}@{step} (config)") continue if n_words is not None: got = checkpoint_n_words(repo, rid) if got is not None and got != int(n_words): rejected.append(f"{rid}@{step} (tokens {got}!={int(n_words)})") continue if best is None or (st.get("updated_at") or 0) > (best[1].get("updated_at") or 0): best = (rid, st) if log: if best: log(f" resume: {stage} -> {best[0]} at step {best[1].get('step')}" + (f" (skipped: {', '.join(rejected)})" if rejected else "")) elif rejected: log(f" resume: {stage} has no compatible run (found {', '.join(rejected)})") return best or (None, None) #: Which run of each stage is *served*. Written locally, mirrored to the Hub. ACTIVE_FILE = "/marimo/runs/shared/active.json" def read_active(repo_id: str, token=None, root: str = "/marimo/runs"): """{stage: {"run_id": ..., "step": ...}} -- the chosen run per stage. Local copy first (a restore always leaves one), Hub second, so a fresh sandbox picks the selection up instead of recomputing it. """ local = Path(root) / "shared" / "active.json" if local.exists(): try: return json.loads(local.read_text()) except Exception: pass return RemoteRepo(repo_id, token=token).read_json("shared/active.json") or {} def active_run_id(repo_id: str, stage: str, token=None, root: str = "/marimo/runs"): entry = read_active(repo_id, token, root).get(stage) if isinstance(entry, dict): return entry.get("run_id") return entry if isinstance(entry, str) else None def set_active_run(repo_id: str, stage: str, run_id: str, step=None, token=None, root: str = "/marimo/runs", log=None): """Record which run of `stage` should be served, locally and on the Hub. "Newest" is a poor rule for serving, and it was the rule: a run that has just been started has a fresher `updated_at` and a *worse* model, so inference silently preferred a barely-trained checkpoint the moment it appeared. Which checkpoint is served is a decision, so it is written down and read back by `open_stage_run` (and through it by `omni.infer`). """ sel = read_active(repo_id, token, root) sel[stage] = {"run_id": run_id, "step": step, "at": time.time()} path = Path(root) / "shared" / "active.json" path.parent.mkdir(parents=True, exist_ok=True) atomic_write_json(path, sel) repo = RemoteRepo(repo_id, token=token) if repo.enabled: res = repo.commit([(str(path), "shared/active.json")], f"served {stage}: {run_id}") if log and not res: log(f" (could not mirror the selection to the Hub: {repo._errors[-1:]})") if log: log(f" served {stage} -> {run_id}" + (f" at step {step}" if step is not None else "")) return sel def open_stage_run(repo_id: str, stage: str, token=None, root="/marimo/runs", ckpt_every_steps=250, ckpt_every_seconds=900.0, honour_active=True): """A RunStore pointing at the run of `stage` to use (None if there is none). `shared/active.json` wins when it names a run for this stage: the newest run is not the best run, and letting the newest win means a just-started run takes over serving. Otherwise the newest `runs/-*/meta/state.json` is used. """ found = None if honour_active: rid = active_run_id(repo_id, stage, token, root) if rid: state = RemoteRepo(repo_id, token=token).read_json(f"runs/{rid}/meta/state.json") if not state: local = Path(root) / rid / "meta" / "state.json" state = json.loads(local.read_text()) if local.exists() else None if state: found = (rid, state) if not found: found = find_latest_stage_run(repo_id, stage, token) if not found: return None, None run_id, state = found store = RunStore(run_id, repo=RemoteRepo(repo_id, token=token, private=CKPT_PRIVATE), root=root, cfg=state.get("config") or {}, ckpt_every_steps=ckpt_every_steps, ckpt_every_seconds=ckpt_every_seconds) return store, state def state_dict_of(m): """Accepts a module, an optimizer, or a raw dict of tensors.""" if isinstance(m, dict): return m return m.state_dict() def load_state_dict(m, sd): if hasattr(m, "load_state_dict"): m.load_state_dict(sd) return m def merge_shards(shards, prefix: str = ""): """Flatten an optimizer state_dict into {dotted.path: tensor}. Optimizer state is nested two deep (state -> param id -> moment), so this recurses instead of assuming a single level. """ out = {} if isinstance(shards, torch.Tensor): out[prefix.rstrip(".")] = shards elif isinstance(shards, dict): for k, v in shards.items(): out.update(merge_shards(v, f"{prefix}{k}.")) elif isinstance(shards, (list, tuple)): for i, v in enumerate(shards): out.update(merge_shards(v, f"{prefix}{i}.")) return out #: Substrings that mean "try again": the Hub is rate limiting us, or the network #: flaked. Anything else is deterministic, and retrying it only delays the caller #: -- which matters most in the shutdown path, where the box is about to vanish. TRANSIENT_ERRORS = ("429", "500", "502", "503", "504", "timeout", "timed out", "connection", "temporarily", "rate limit") #: A *hard* ceiling, not a blip: the account may create only 128 commits per hour. #: It contains the words "rate limit", so it would otherwise be swallowed by #: TRANSIENT_ERRORS and slept through -- one backup cycle was observed sitting in #: backoff for 23 minutes with nothing on the wire, which looks exactly like a #: healthy process doing its job. It is checked *before* TRANSIENT_ERRORS. HARD_LIMIT = ("per hour", "commit creation") #: ...and this one means "already done": the Hub holds exactly these bytes. NO_CHANGE = ("No files have been modified", "no files have been modified", "already up to date") # ------------------------------------------------------------------ commit budget #: The Hub allows 128 *commit creations* per hour, and enforcing it is a 429 that is #: expensive to discover -- one cycle was found sitting in backoff for 23 minutes, #: looking healthy. So the budget is tracked here instead, shared by every writer in #: every process (trainers, the backup daemon, the recovery paths) through a file, #: because a per-process limit would be no limit at all. It is set below the real #: ceiling on purpose: the remainder is headroom for `priority=True` pushes, which is #: what a box about to be reaped uses. COMMIT_BUDGET_PER_HOUR = 100 COMMIT_LOG = Path("/marimo/logs/commits.jsonl") def commit_allowance(now: float | None = None) -> tuple[bool, int, int]: """(may commit, commits in the last hour, budget).""" now = now or time.time() n = 0 try: for line in COMMIT_LOG.read_text().splitlines()[-600:]: try: if now - float(json.loads(line).get("t", 0)) < 3600: n += 1 except Exception: continue except Exception: pass return n < COMMIT_BUDGET_PER_HOUR, n, COMMIT_BUDGET_PER_HOUR def note_commit(what: str = ""): """Record a commit. Append-only, one line, so concurrent writers are safe.""" try: COMMIT_LOG.parent.mkdir(parents=True, exist_ok=True) with open(COMMIT_LOG, "a") as fh: fh.write(json.dumps({"t": time.time(), "what": what[:80]}) + "\n") except Exception: pass class RemoteRepo: """Thin wrapper over the HF Hub, with retries and a background uploader.""" def __init__(self, repo_id: str, token: str | None = None, private: bool = CKPT_PRIVATE, enabled: bool = True): self.repo_id = repo_id self.token = token self.private = private self.enabled = enabled and bool(repo_id) self._api = None self._q: queue.Queue = queue.Queue() self._thread = None self._errors: list[str] = [] #: epoch seconds until which the account is refusing commit creation self.rate_limited_until: float = 0.0 # -- lazy import so the notebook can run without huggingface_hub --------- def api(self): if self._api is None: import huggingface_hub as hf hf.create_repo(self.repo_id, repo_type="model", private=self.private, exist_ok=True, token=self.token) self._api = hf.HfApi(token=self.token) return self._api def _already_committed(self, message: str, look_back: int = 6) -> bool: """True if a commit with this exact message is already in the repo. A transient error does not mean the commit failed: an upload that lands but times out waiting for its *response* looks identical from here. Retrying then re-uploads multi-gigabyte files and spends another commit -- six identical `resync 4 run(s)` commits inside six seconds is what that looked like. The listing costs one small request and saves a multi-gigabyte one. """ try: if time.time() - getattr(self, "_titles_at", 0.0) > 30.0: from huggingface_hub import HfApi self._recent_titles = { c.title for c in HfApi(token=self.token).list_repo_commits( self.repo_id, repo_type="model")[:look_back]} self._titles_at = time.time() return message in self._recent_titles except Exception: return False def commit(self, items, message: str = "checkpoint", retries: int = 6, priority: bool = False): """Upload many files in ONE commit. The Hub counts API *requests*, not bytes: a checkpoint made of model+optim+meta+log+state used to cost five calls and blew the per-hour API quota. One commit per checkpoint costs one. """ if not self.enabled or not items: return None from huggingface_hub import CommitOperationAdd, create_commit ops = [CommitOperationAdd(path_in_repo=rp, path_or_fileobj=str(lp)) for lp, rp in items if Path(lp).exists()] if not ops: return None if not priority: allowed, used, budget = commit_allowance() if not allowed: self._errors.append( f"{time.time():.0f} commit deferred: {used}/{budget} commits used " f"in the last hour, keeping headroom for a final flush") return None if time.time() < self.rate_limited_until: self._errors.append( f"{time.time():.0f} commit deferred: hourly commit ceiling, " f"{self.rate_limited_until - time.time():.0f}s left") return None self.api() for attempt in range(retries): try: res = create_commit(repo_id=self.repo_id, operations=ops, token=self.token, commit_message=message) note_commit(message) return res except Exception as e: msg = str(e) if any(s in msg for s in NO_CHANGE): # Idempotent: this is the recovery path's normal case (the # uploader already got there). Returning a truthy sentinel # keeps callers from reporting a spurious failure -- and # returning immediately avoids a multi-minute retry stall. return "no-change" self._errors.append(f"{time.time():.0f} commit: {type(e).__name__} {msg[:200]}") if any(s in msg for s in HARD_LIMIT): # Give up at once and remember. Callers should batch rather than # wait: one commit per *cycle* instead of one per run is the fix, # not a longer sleep. self.rate_limited_until = time.time() + 3600.0 return None if attempt == retries - 1 or not any(s in msg for s in TRANSIENT_ERRORS): return None if self._already_committed(message): note_commit(message) return "already-committed" wait = 30.0 * (attempt + 1) if "429" in msg: m = re.search(r"[Rr]etry after (\d+)", msg) wait = float(m.group(1)) + 5 if m else 120.0 if wait > 180.0: self.rate_limited_until = time.time() + wait return None time.sleep(min(wait, 180.0)) return None def upload(self, local_path, repo_path: str, message: str = "checkpoint"): if not self.enabled: return None from huggingface_hub import upload_file for attempt in range(3): try: return upload_file(path_or_fileobj=str(local_path), path_in_repo=repo_path, repo_id=self.repo_id, token=self.token, commit_message=message) except Exception as e: # network flake inside the sandbox msg = str(e) if any(s in msg for s in NO_CHANGE): return "no-change" self._errors.append(f"{time.time():.0f} {repo_path}: {type(e).__name__} {msg[:160]}") time.sleep(2 * (attempt + 1)) return None def download(self, repo_path: str, local_path) -> bool: if not self.enabled: return False from huggingface_hub import hf_hub_download try: p = hf_hub_download(repo_id=self.repo_id, filename=repo_path, token=self.token, force_download=False) except Exception: return False data = Path(p).read_bytes() atomic_write_bytes(local_path, data) return True def read_json(self, repo_path: str): if not self.enabled: return None from huggingface_hub import hf_hub_download try: p = hf_hub_download(repo_id=self.repo_id, filename=repo_path, token=self.token) return json.loads(Path(p).read_text()) except Exception: return None # -- background uploader ------------------------------------------------- def start(self): if not self.enabled or self._thread is not None: return def _loop(): while True: item = self._q.get() if item is None: return payload, message = item if isinstance(payload, list): self.commit(payload, message) else: self.upload(payload[0], payload[1], message) self._thread = threading.Thread(target=_loop, daemon=True, name="hub-uploader") self._thread.start() def submit(self, local_path, repo_path: str, message: str = "ckpt"): if not self.enabled: return self.start() self._q.put(((str(local_path), repo_path), message)) def submit_commit(self, items, message: str = "checkpoint"): """Queue one atomic commit for a whole checkpoint.""" if not self.enabled: return self.start() self._q.put(([(str(a), b) for a, b in items], message)) def drain(self, timeout: float = 300.0): """Block until the uploader is idle (used at shutdown).""" if not self.enabled or self._thread is None: return t0 = time.time() while not self._q.empty() and time.time() - t0 < timeout: time.sleep(0.5) #: The metric that decides "best" for each stage, and the direction that is better #: (all of these are loss-like, so lower is better). Used to pin a checkpoint for #: inference and to protect it from pruning. #: The metric that decides which checkpoint gets pinned as "best" and protected from #: pruning, per stage. Two ways this was wrong, both silent: #: #: * the keys were `"ctc"` and `"stft"` while the stages log `"ctc_loss"` and #: `"stft_loss"`, so `metrics.get(key)` was always `None` and the protection #: **never fired** for the ear or the vocoder. That is why the vocoder's best #: weights (stft 0.82) were pruned away in favour of worse, newer ones; #: * `speech` used `"loss"`, but that stage logs `loss` on *two kinds of batch* -- #: a vision-language batch (~0.08) and a speech batch (~0.72). Comparing them pins #: an arbitrary step: it chose step 39002 because a VL batch briefly scored 0.081, #: leaving inference serving a checkpoint 9 500 steps behind the trainer. #: #: So the key must name a metric that only appears on batches that are *comparable* #: with each other. `mel_loss` appears only on speech batches. BEST_METRIC = {"vlm": "loss", "captions": "loss", "speech": "mel_loss", "ear": "ctc_loss", "vocoder": "stft_loss"} class RunStore: """Local run directory + remote mirror. Layout (local root/runs// = remote runs//): meta/state.json stage progress: step, samples, wall, best metric latest/.pt rolling latest payload per tag (model, optim, ...) mile-/ kept milestone payloads (default: newest 3) log.jsonl metrics, append-only train.log raw stdout/stderr of the trainer """ def __init__(self, run_id: str, repo: RemoteRepo | None = None, root="/marimo/runs", cfg: dict | None = None, ckpt_every_steps: int = 200, ckpt_every_seconds: float = 600.0, keep_milestones: int = 3, milestone_every: int = 5, push_every_seconds: float = 1200.0): self.run_id = run_id self.repo = repo self.cfg = cfg or {} self.cfg_hash = config_hash(self.cfg) self.run_dir = Path(root) / run_id self.latest_dir = self.run_dir / "latest" self.state_path = self.run_dir / "meta" / "state.json" self.log_path = self.run_dir / "log.jsonl" self.train_log = self.run_dir / "train.log" self.ckpt_every_steps = ckpt_every_steps self.ckpt_every_seconds = ckpt_every_seconds self.keep_milestones = keep_milestones self.milestone_every = milestone_every self._last_ckpt_t = time.time() # 0.0 would read as "infinitely overdue" self._last_push_t = time.time() # Hub commits are rate limited, so self.push_every_seconds = push_every_seconds # they get their own cadence self._n_ckpt = 0 self._lock = threading.Lock() self.run_dir.mkdir(parents=True, exist_ok=True) self.latest_dir.mkdir(parents=True, exist_ok=True) # NOTE: `close` is registered on the first *save*, not here. Registering it # for every store -- including the ones opened merely to ask what exists -- # meant that reading an old run bumped its `updated_at` on exit, and # `find_latest_stage_run` sorts by exactly that. Inspecting a run made it # look like the newest one, so inference served a 6 502-step trunk while a # 47 502-step one was training. See `close` for the rest of the story. # ---------------------------------------------------------------- remote def rpath(self, *parts): return "/".join(["runs", self.run_id, *[str(p) for p in parts]]) def push_state(self): if self.repo: self.repo.submit(self.state_path, self.rpath("meta", "state.json"), "state") def pull_state(self): """Fetch remote state.json (after a sandbox death, local copy is gone).""" if not self.repo: return None st = self.repo.read_json(self.rpath("meta", "state.json")) if st: atomic_write_json(self.state_path, st) # keep local copy in sync return st # ----------------------------------------------------------------- state def state(self) -> dict: try: return json.loads(self.state_path.read_text()) except Exception: return {} def set_progress(self, **kw): st = self.state() st.setdefault("run_id", self.run_id) st.setdefault("cfg_hash", self.cfg_hash) st.update(kw) st["updated_at"] = time.time() atomic_write_json(self.state_path, st) return st def log(self, **metrics): rec = {"t": time.time(), "wall": time.strftime("%H:%M:%S"), **metrics} with self._lock: with open(self.log_path, "a") as f: f.write(json.dumps(rec, default=str) + "\n") print("METRIC " + json.dumps({k: (round(v, 5) if isinstance(v, float) else v) for k, v in rec.items()}), flush=True) return rec def read_log(self, limit: int = 4000) -> list[dict]: if not self.log_path.exists(): return [] out = [] for line in self.log_path.read_text().splitlines()[-limit:]: try: out.append(json.loads(line)) except Exception: pass return out def tail_train_log(self, limit: int = 60) -> str: if not self.train_log.exists(): return "" return "\n".join(self.train_log.read_text(errors="replace").splitlines()[-limit:]) # ------------------------------------------------------------ checkpoints def should_checkpoint(self, step: int, force: bool = False) -> bool: if force: return True if step - self._n_ckpt >= self.ckpt_every_steps: return True return (time.time() - self._last_ckpt_t) >= self.ckpt_every_seconds def save(self, *, stage: str, step: int, payload: dict, metrics: dict | None = None, milestone: bool | None = None, sync: bool = True): """payload: {tag: module|optimizer|dict-of-tensors|jsonable}""" t0 = time.time() tags = {} for tag, obj in payload.items(): f = self.latest_dir / f"{tag}.pt" if isinstance(obj, (dict, list)) and all(not isinstance(v, torch.Tensor) for v in (obj.values() if isinstance(obj, dict) else obj)): atomic_write_json(f, obj) else: atomic_torch_save(f, obj) tags[tag] = f.name push_miles = [] if milestone is None: # keep every `milestone_every`-th checkpoint as a pinned milestone milestone = (step // max(self.ckpt_every_steps, 1)) % self.milestone_every == 1 if milestone: md = self.run_dir / f"mile-{step:08d}" md.mkdir(parents=True, exist_ok=True) for tag, name in tags.items(): atomic_write_bytes(md / name, (self.latest_dir / name).read_bytes()) push_miles.append(md) self.prune_milestones() st = self.set_progress(stage=stage, step=step, local_ckpt_time=time.time(), **(metrics or {})) st["tags"] = tags st["cfg_hash"] = self.cfg_hash if metrics: st["metrics"] = metrics # --- pin the best checkpoint --------------------------------------- # The best weights are what inference serves and what nobody wants to lose, # so a new best becomes a milestone of its own (whatever the cadence), is # pinned in active.json, and is protected by prune_milestones. Keeping only # the newest few by age is how the vocoder's best run was deleted. key = BEST_METRIC.get(stage, "loss") value = (metrics or {}).get(key) if isinstance(value, (int, float)): best = dict(st.get("best") or {}) cur = best.get(key) if cur is None or float(value) < float(cur): md = self.run_dir / f"mile-{step:08d}" if not md.exists(): md.mkdir(parents=True, exist_ok=True) for tag, name in tags.items(): atomic_write_bytes(md / name, (self.latest_dir / name).read_bytes()) if not any(str(p) == str(md) for p in push_miles): push_miles.append(md) best[key] = float(value) best["step"] = step st["best"] = best atomic_write_json(self.state_path, st) self.set_active(step) milestone = milestone or True atomic_write_json(self.state_path, st) if not self._n_ckpt: atexit.register(self.close) # first real checkpoint -> flush on exit self._n_ckpt, self._last_ckpt_t = step, time.time() want_push = (time.time() - self._last_push_t) >= self.push_every_seconds if not want_push and (milestone or step >= self.cfg.get("_end_step", 10 ** 9)): want_push = True if sync and self.repo and want_push: self._last_push_t = time.time() # payload first, pointer last: remote state.json implies payload ok. # The optimizer is 2x the model size; pushing it every checkpoint # starves the uploader, so it rides along on milestones only (the # interrupt handler resyncs it from the local run dir if needed). items = [(self.latest_dir / name, self.rpath("latest", name)) for tag, name in tags.items() if tag != "optim" or milestone] for md in push_miles: items += [(md / name, self.rpath(md.name, name)) for tag, name in tags.items()] items.append((self.log_path, self.rpath("log.jsonl"))) items.append((self.state_path, self.rpath("meta", "state.json"))) # pointer # `active.json` names the checkpoint inference serves; without it a # restored box would serve whatever step happens to be newest active = self.run_dir / "active.json" if active.exists(): items.append((active, self.rpath("active.json"))) self.repo.submit_commit(items, f"{stage} step {step}") return self.state_path, round(time.time() - t0, 2) def pinned_steps(self) -> set: """Every step named as "the one to serve" for this run. Two files can name it, they are written by different things, and they disagreed here: * `active.json` *in the run directory* -- written by `set_active` whenever the run records a best *training* metric. This is the one `load_for_inference` honours, so it is what decides the served step. * `shared/active.json` -- written by `select_served.py`, the cross-run selection. `open_stage_run` reads it to choose *which run* serves a stage, not which step. The speech run pinned 51 002 locally while the shared selection named 39 002 -- the checkpoint measured at **47.0%** exact match against **35.5%** -- and pruning kept only the former, deleting the better one. It was recoverable at all only because the Hub archives every milestone that was ever pushed. """ steps = set() try: steps.add(int(json.loads((self.run_dir / "active.json").read_text())["step"])) except Exception: pass try: sel = json.loads((self.run_dir.parent / "shared" / "active.json").read_text()) for entry in sel.values(): if (isinstance(entry, dict) and entry.get("run_id") == self.run_id and entry.get("step") is not None): steps.add(int(entry["step"])) except Exception: pass return steps def prune_milestones(self): """Keep the newest few milestones -- and *always* every pinned one. Pruning by age alone silently destroyed checkpoints that were being served: the vocoder's best (`stft` 0.82) and the speech trunk measured at 47% exact match. Having produced a served model is a reason to survive. """ import shutil keep = {f"mile-{s:08d}" for s in self.pinned_steps()} ms = sorted(self.run_dir.glob("mile-*"), key=lambda p: p.name) removable = [p for p in ms if p.name not in keep] for p in removable[: max(0, len(removable) - self.keep_milestones)]: shutil.rmtree(p, ignore_errors=True) return sorted(p.name for p in ms if p.exists()) def schedule_total(self, total: int) -> int: """The horizon the LR schedule is shaped for, recorded on first use. A cosine keyed on the launcher's `--steps` is silently *re-heated* every time somebody raises the budget: at the same step the learning rate jumps back towards its initial value and knocks a converged model off its minimum. Recording the horizon in the run's own state means every later resume is shaped by the same curve the run started with. """ st = self.state() if not st.get("schedule_total"): st["schedule_total"] = int(total) atomic_write_json(self.state_path, st) return int(st["schedule_total"]) def budget_note(self, steps: int) -> str: """A warning when the launcher asks for more steps than the run planned.""" rec = self.state().get("schedule_total") if rec and int(steps) > int(rec): return (f"note: this run's LR schedule was shaped for {rec} steps and the " f"launcher now asks for {steps}. The learning rate has already " f"decayed, so continuing is mostly fine-tuning; a new run is the " f"honest way to train longer.") return "" def set_active(self, step, tag: str = "model"): """Pin a milestone for *inference* (model selection), not for resume. `latest` stays the training-resume state (model+optimizer in sync); this only tells inference which weights to serve. """ atomic_write_json(self.run_dir / "active.json", {"step": step, "tag": tag}) return self.run_dir / "active.json" def active_checkpoint(self, tag: str = "model"): p = self.run_dir / "active.json" if not p.exists(): return None, None info = json.loads(p.read_text()) f = self.run_dir / f"mile-{int(info['step']):08d}" / f"{tag}.pt" return (f, info) if f.exists() else (None, None) def load_for_inference(self, tag: str = "model"): """Best checkpoint (if model selection ran), else the newest one.""" f, info = self.active_checkpoint(tag) if f is not None: try: return {"step": int(info["step"]), "payload": torch.load(f, map_location="cpu", weights_only=False), "active": True, "from_remote": False, "state": self.state()} except Exception: pass return self.load_latest(tag=tag) def load_latest(self, stage: str | None = None, tag: str = "model", remote=True) -> dict | None: """Return {'step', 'stage', 'payload': ..., 'state': ...} or None.""" state = self.state() f = self.latest_dir / f"{tag}.pt" pulled = False if not f.exists() and remote and self.repo: if self.repo.download(self.rpath("latest", f"{tag}.pt"), f): pulled = True if not f.exists(): return None if not state or pulled: rs = self.pull_state() if rs: state = rs try: payload = torch.load(f, map_location="cpu", weights_only=False) if f.suffix == ".pt" else json.loads(f.read_text()) except Exception: payload = json.loads(f.read_text()) if f.suffix == ".pt" else None return {"step": int(state.get("step", 0)), "stage": state.get("stage"), "state": state, "payload": payload, "from_remote": pulled} def close(self): """Flush the pointer on exit -- but only if this process actually saved. A read-only store has nothing to close. That distinction was missing, and it was expensive: `close` writes `closed_at` and therefore bumps `updated_at`, which is the field `find_latest_stage_run` sorts by. So *looking* at an old run promoted it to "newest", and `OmniSystem.load` served a stale trunk -- the published eval number was remeasured and came back identical, because it was scoring the same 6 502-step checkpoint both times. It also spent a Hub commit labelled "final state" on every inspection. """ if self.repo and self._n_ckpt: try: self.set_progress(closed_at=time.time()) self.repo.submit(self.state_path, self.rpath("meta", "state.json"), "final state") self.repo.drain(timeout=180) except Exception: pass def _remote_files(repo): try: return repo.api().list_repo_files(repo.repo_id) except Exception: return [] def resync_local_runs(repo_id: str, token: str | None = None, root="/marimo/runs", only_stages=None, log=print, retries: int = 4, include_optim: bool = True, force: bool = False, batch: bool = True, priority: bool = False): """Push whatever local checkpoints are newer than what the Hub holds. The trainer's uploads are fire-and-forget on a background thread: if the network stalls (or the sandbox is about to be reaped), the local run dir is still ahead. This is the recovery path, and it is idempotent. `batch=True` puts every run's payload into a *single* commit. The account may create only 128 commits per hour, and one cycle with six live runs used to spend six of them -- which is how the ceiling gets hit and then, worse, *slept through*. A commit is atomic and the run's `state.json` is still ordered last inside it, so the "state.json implies the payload arrived" invariant holds even more strongly than when each run was its own commit. """ from pathlib import Path as _Path repo = RemoteRepo(repo_id, token=token, private=CKPT_PRIVATE) if time.time() < repo.rate_limited_until: log(f" resync deferred: hourly commit ceiling, " f"{repo.rate_limited_until - time.time():.0f}s left") return [] remote_files = _remote_files(repo) # one listing for all runs, not one each todo: list[tuple[str, dict, dict, list]] = [] for d in sorted(_Path(root).glob("*")): state_file = d / "meta" / "state.json" if not state_file.exists(): continue run_id = d.name if only_stages and run_id.split("-")[0] not in only_stages: continue local = json.loads(state_file.read_text()) remote = repo.read_json(f"runs/{run_id}/meta/state.json") or {} # `need` is decided per run. The optimizer is why the check exists at all: # the state pointer can already be current while the (much larger) optimizer # payload never made it. It must not be folded back into `force` -- that # leaks: one optimizer-less run (a brand-new one, say) would then force # *every* later run to re-hash and re-upload its full checkpoint, which is # what the Hub's per-hour commit quota notices first. need = force if not need and include_optim and (d / "latest" / "optim.pt").exists(): # only meaningful if there is an optimizer here to upload: a stage like # `align` never writes one, and asking the Hub for it every cycle would # spend a commit on nothing, forever. need = not any(f == f"runs/{run_id}/latest/optim.pt" for f in remote_files) if not need and local.get("step", 0) <= remote.get("step", 0) \ and local.get("updated_at", 0) <= remote.get("updated_at", 0): continue items = [] tags = ("model", "optim", "meta") if include_optim else ("model", "meta") for tag in tags: f = d / "latest" / f"{tag}.pt" if f.exists(): items.append((f, f"runs/{run_id}/latest/{tag}.pt")) if (d / "log.jsonl").exists(): items.append((d / "log.jsonl", f"runs/{run_id}/log.jsonl")) # The pinned checkpoint is what a restored box would *serve*, so the pin and # the weights it names travel together -- otherwise a fresh sandbox serves # whichever step happens to be newest, which is not the same model. active = d / "active.json" if active.exists(): try: pin = int(json.loads(active.read_text()).get("step")) except Exception: pin = None if pin is not None: md = d / f"mile-{pin:08d}" for tag in ("model", "meta"): f = md / f"{tag}.pt" dest = f"runs/{run_id}/{md.name}/{tag}.pt" if f.exists() and all(b != dest for _a, b in items): items.append((f, dest)) items.append((active, f"runs/{run_id}/active.json")) items.append((state_file, f"runs/{run_id}/meta/state.json")) # pointer last todo.append((run_id, local, remote, items)) if not todo: return [] done = [] if batch: everything = [it for _r, _l, _rm, items in todo for it in items] summary = ", ".join(f"{r}@{l.get('step')}" for r, l, _rm, _i in todo) if repo.commit(everything, f"resync {len(todo)} run(s): {summary}", retries=retries, priority=priority): for run_id, local, remote, _items in todo: log(f" resynced {run_id}: local step {local.get('step')} " f"(hub had {remote.get('step')})") done.append(run_id) return done # One commit for every run is the cheap path, but it is also a single point of # failure: a request carrying ~15 GB can simply be refused, and then *nothing* # is saved while the caller prints a warning into a log nobody is reading -- # the exact silent-failure shape this project keeps producing. Falling back to # one commit per run uses smaller requests, and one bad run cannot take the # rest of the work down with it. log(f" batched resync of {len(todo)} run(s) refused - retrying one run at a time") for run_id, local, remote, items in todo: if not repo.commit(items, f"resync {run_id} step {local.get('step')}", retries=retries): log(f" {run_id}: push failed (rate limit?) - local state is still current") continue log(f" resynced {run_id}: local step {local.get('step')} (hub had {remote.get('step')})") done.append(run_id) return done return @app.cell(hide_code=True) def lib_text(): # ===== omni/text.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/text.py using only the imports this module actually needs. SPECIALS = ["", "", "", "", ""] # 39-phone ARPAbet inventory (stress digits stripped) PHONES = ["AA", "AE", "AH", "AO", "AW", "AY", "B", "CH", "D", "DH", "EH", "ER", "EY", "F", "G", "HH", "IH", "IY", "JH", "K", "L", "M", "N", "NG", "OW", "OY", "P", "R", "S", "SH", "T", "TH", "UH", "UW", "V", "W", "Y", "Z", "ZH"] VOWELS = {"AA", "AE", "AH", "AO", "AW", "AY", "EH", "ER", "EY", "IH", "IY", "OW", "OY", "UH", "UW"} _WORD_RE = re.compile(r"[a-z0-9']+") def words_of(text: str) -> list[str]: return _WORD_RE.findall(text.lower()) class WordVocab: """Deterministic word-level vocabulary. One vocab builds train AND inference.""" def __init__(self, itos: list[str]): self.itos = list(itos) self.stoi = {w: i for i, w in enumerate(self.itos)} self.pad = self.stoi[""] self.bos = self.stoi[""] self.eos = self.stoi[""] self.sep = self.stoi[""] def __len__(self): return len(self.itos) @classmethod def build(cls, texts, min_count: int = 1, extra=()): from collections import Counter c = Counter() for t in texts: c.update(words_of(t)) toks = list(SPECIALS) for w in sorted(extra): if w not in c: c[w] = 0 for w, n in sorted(c.items()): if n >= min_count and w not in SPECIALS: toks.append(w) return cls(toks) def encode(self, text: str, add_bos=False, add_eos=False) -> list[int]: ids = [self.stoi.get(w, self.stoi[""]) for w in words_of(text)] if add_bos: ids = [self.bos] + ids if add_eos: ids = ids + [self.eos] return ids def decode(self, ids) -> str: out = [] for i in ids: i = int(i) w = self.itos[i] if 0 <= i < len(self.itos) else "?" if w in ("", ""): break if w in SPECIALS: continue out.append(w) return " ".join(out) def save(self, path): Path(path).write_text(json.dumps(self.itos)) @classmethod def load(cls, path): return cls(json.loads(Path(path).read_text())) class PhoneVocab: def __init__(self, phones: list[str] = None): phones = list(phones or PHONES) self.itos = ["", "", "", "", ""] + phones self.stoi = {p: i for i, p in enumerate(self.itos)} self.pad = self.stoi[""] self.bos = self.stoi[""] self.eos = self.stoi[""] self.blank = self.stoi[""] # CTC blank self.unk = self.stoi[""] self.phone_offset = 5 def __len__(self): return len(self.itos) def encode(self, phones) -> list[int]: return [self.stoi.get(p, self.unk) for p in phones] def decode(self, ids, keep_stress=False) -> list[str]: out = [] for i in ids: i = int(i) p = self.itos[i] if 0 <= i < len(self.itos) else "" if p in ("", "", "", ""): continue if out and out[-1] == p: continue out.append(p) return out def save(self, path): Path(path).write_text(json.dumps(self.itos)) @classmethod def load(cls, path): return cls(json.loads(Path(path).read_text())[5:]) # --- crude but deterministic letter-to-sound for out-of-vocabulary words ----- _LTS = { "a": "AH", "b": "B", "c": "K", "d": "D", "e": "EH", "f": "F", "g": "G", "h": "HH", "i": "IH", "j": "JH", "k": "K", "l": "L", "m": "M", "n": "N", "o": "OW", "p": "P", "q": "K", "r": "R", "s": "S", "t": "T", "u": "UW", "v": "V", "w": "W", "x": "K S", "y": "Y", "z": "Z", } class G2P: """CMUdict lookup + letter-to-sound fallback.""" def __init__(self, dict_path: str | None = None): self.table: dict[str, list[str]] = {} if dict_path and Path(dict_path).exists(): for line in Path(dict_path).read_text(errors="replace").splitlines(): line = line.split("#", 1)[0].strip() if not line: continue parts = line.replace("(", " ").replace(")", " ").split() if len(parts) < 2: continue w = parts[0].lower().strip("'") ph = [re.sub(r"\d", "", p).upper() for p in parts[1:]] ph = [p for p in ph if p in PHONES] if ph and w not in self.table: self.table[w] = ph def __len__(self): return len(self.table) def word_to_phones(self, w: str) -> list[str]: w = w.lower() if w in self.table: return list(self.table[w]) if w.isdigit(): words = _number_words(int(w)) out = [] for x in words: out += self.word_to_phones(x) return out out = [] for ch in w: if ch in _LTS: out += _LTS[ch].split() return out or [PHONES[0]] def text_to_phones(self, text: str) -> list[str]: out = [] for w in words_of(text): out += self.word_to_phones(w) return out _NUM = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"] _TENS = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"] def _number_words(n: int) -> list[str]: if n < 20: return [_NUM[n]] if n < 100: return [_TENS[n // 10]] + ([] if n % 10 == 0 else [_NUM[n % 10]]) if n < 1000: rest = _number_words(n % 100) if n % 100 else [] return [_NUM[n // 100], "hundred"] + rest return [_NUM[0]] return @app.cell(hide_code=True) def lib_scenes(): # ===== omni/scenes.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/scenes.py using only the imports this module actually needs. #: Bumped whenever the generated data changes semantics. Part of a run's #: identity (see omni/config.RunConfig): a checkpoint trained on v1 is not a #: checkpoint trained on v2, even with the same seed. DATA_VERSION = 2 SHAPES = ["circle", "square", "triangle", "star", "hexagon", "diamond"] COLORS = { "red": (222, 68, 62), "green": (72, 181, 96), "blue": (72, 110, 232), "yellow": (240, 200, 52), "cyan": (72, 200, 216), "magenta": (216, 84, 196), "orange": (246, 148, 48), "purple": (146, 88, 224), "white": (246, 246, 246), "black": (28, 28, 32), "gray": (152, 156, 164), "brown": (156, 104, 60), } BG = {"dark": (22, 24, 32), "light": (240, 238, 230), "slate": (58, 62, 78), "cream": (250, 240, 214)} SIZE_R = {"small": 11.0, "medium": 16.0, "large": 22.0} SIZE_WORD = {"small": ["small", "little", "tiny"], "medium": ["medium", "mid sized", "middle sized"], "large": ["large", "big", "huge"]} NUM_WORD = ["zero", "one", "two", "three", "four", "five", "six", "seven"] GRID = [(0.22, 0.22), (0.5, 0.22), (0.78, 0.22), (0.22, 0.5), (0.5, 0.5), (0.78, 0.5), (0.22, 0.78), (0.5, 0.78), (0.78, 0.78)] H_POS = ["left", "center", "right"] V_POS = ["top", "middle", "bottom"] def _mask(shape, X, Y, cx, cy, r): dx, dy = X - cx, Y - cy if shape == "circle": return dx * dx + dy * dy <= r * r if shape == "square": return (np.abs(dx) <= r) & (np.abs(dy) <= r) if shape == "diamond": return (np.abs(dx) + np.abs(dy)) <= r * 1.25 if shape == "triangle": # downward-pointing triangle (image y grows downward) k = np.clip((dy + r) / (2 * r), 0, 1) half = r * k return (dy >= -r) & (dy <= r) & (np.abs(dx) <= half) if shape == "hexagon": a = np.arctan2(dy, dx) rr = r * (np.cos(np.pi / 6) / np.maximum(np.cos((a % (np.pi / 3)) - np.pi / 6), 1e-3)) return (dx * dx + dy * dy) <= rr * rr if shape == "star": a = np.arctan2(dy, dx) rr = r * (1.0 - 0.45 * np.cos(5 * a)) return (dx * dx + dy * dy) <= rr * rr raise ValueError(shape) def render_scene(scene: dict, px: int = 128, ss: int = 2) -> np.ndarray: """Rasterise a scene dict to an (px, px, 3) uint8 image.""" n = px * ss img = np.zeros((n, n, 3), np.float32) img[:] = BG[scene["bg"]] ys = (np.arange(n) + 0.5) / n * px X, Y = np.meshgrid(ys, ys) for ob in scene["objects"]: cx, cy = ob["cx"] * px, ob["cy"] * px r = SIZE_R[ob["size"]] * (1.0 + 0.12 * ob.get("jitter", 0.0)) m = _mask(ob["shape"], X, Y, cx, cy, r) col = np.array(COLORS[ob["color"]], np.float32) shade = 1.0 + 0.18 * (Y - cy) / max(r, 1.0) # cheap directional shading img[m] = np.clip(col[None, :] * shade[m][:, None], 0, 255) if ss > 1: img = img.reshape(px, ss, px, ss, 3).mean(axis=(1, 3)) return np.clip(img, 0, 255).astype(np.uint8) def make_scene(rng, nmin: int = 1, nmax: int = 4) -> dict: n = int(rng.integers(nmin, nmax + 1)) cells = list(rng.permutation(len(GRID))[:n]) objects = [] anchor_shape = anchor_color = None for i, c in enumerate(cells): gx, gy = GRID[int(c)] shape = SHAPES[int(rng.integers(len(SHAPES)))] color = list(COLORS)[int(rng.integers(len(COLORS)))] if i == 0: # The first object is the *anchor*: its shape and its colour are kept # unique in the scene, so "what colour is the " and "what shape # is the " always have exactly one answer. Without this the # ground truth for those questions would be ambiguous, and the model # would be asked to guess. anchor_shape, anchor_color = shape, color else: seen_shapes = {o["shape"] for o in objects} seen_colors = {o["color"] for o in objects} while shape == anchor_shape and len(seen_shapes) < len(SHAPES): shape = SHAPES[(SHAPES.index(shape) + 1) % len(SHAPES)] while color == anchor_color and len(seen_colors) < len(COLORS): idx = list(COLORS).index(color) color = list(COLORS)[(idx + 1) % len(COLORS)] objects.append({ "shape": shape, "color": color, "size": list(SIZE_R)[int(rng.integers(3))], "cell": int(c), "cx": float(np.clip(gx + rng.uniform(-0.05, 0.05), 0.15, 0.85)), "cy": float(np.clip(gy + rng.uniform(-0.05, 0.05), 0.15, 0.85)), "jitter": float(rng.uniform(-1, 1)), }) # cells of the same object never collide; enforce non-overlap by dropping return {"bg": list(BG)[int(rng.integers(len(BG)))], "objects": objects} def _pick(rng, items): return items[int(rng.integers(len(items)))] def _plural(word, k): return word if k == 1 else word + "s" def _loc(ob): gx = min(2, int(ob["cx"] * 3)) gy = min(2, int(ob["cy"] * 3)) h, v = H_POS[gx], V_POS[gy] return "center" if (h, v) == ("center", "middle") else f"{v} {h}" def _unique_objects(scene, color=None, shape=None): """Objects whose (colour, shape) signature is unique in the scene. A question that names only those two attributes must have exactly one referent, otherwise the ground truth is ill-defined and the model is being asked to guess. """ out = [] for o in scene["objects"]: if color is not None and o["color"] != color: continue if shape is not None and o["shape"] != shape: continue if len(_distinct_shape(scene, color=o["color"], shape=o["shape"])) == 1: out.append(o) return out def _distinct_shape(scene, color=None, size=None, shape=None): cands = [o for o in scene["objects"] if (color is None or o["color"] == color) and (size is None or o["size"] == size) and (shape is None or o["shape"] == shape)] return cands def question_answer(scene: dict, rng, qtypes: list[str] | None = None): """Return (question, answer) grounded in the scene. Answer is always a short, speakable word sequence.""" obs = scene["objects"] qtypes = qtypes or ["count", "color", "shape", "where", "size", "caption", "total", "compare", "presence", "relation"] sos = _distinct_shape(scene) uniq = _unique_objects(scene) or sos # unambiguous referents for _ in range(12): qt = _pick(rng, qtypes) try: if qt == "count": o = _pick(rng, sos) k = len(_distinct_shape(scene, color=o["color"], shape=o["shape"])) noun = _plural(o["shape"], 2) # always plural in a count question q = _pick(rng, [ f"how many {o['color']} {noun} are there", f"how many {noun} are {o['color']}", f"count the {o['color']} {noun}", f"count the {noun} that are {o['color']}", ]) return q, NUM_WORD[k] if qt == "color": shapes = [sh for sh in sorted({o["shape"] for o in obs}) if len(_distinct_shape(scene, shape=sh)) == 1] if not shapes: continue sh = _pick(rng, shapes) o = _distinct_shape(scene, shape=sh)[0] q = _pick(rng, [f"what color is the {sh}", f"what is the color of the {sh}", f"what colour is the {sh}"]) return q, o["color"] if qt == "shape": colors = [c for c in sorted({o["color"] for o in obs}) if len(_distinct_shape(scene, color=c)) == 1] if not colors: continue col = _pick(rng, colors) o = _distinct_shape(scene, color=col)[0] q = _pick(rng, [f"what shape is the {col} object", f"which shape is {col}", f"name the shape of the {col} thing"]) return q, o["shape"] if qt == "where": o = _pick(rng, uniq) q = _pick(rng, [f"where is the {o['color']} {o['shape']}", f"in which part of the image is the {o['color']} {o['shape']}", f"locate the {o['color']} {o['shape']}"]) return q, _loc(o) if qt == "size": o = _pick(rng, uniq) # 'big or small' offers exactly two choices, so never ask it about # an object whose size is neither (the answer would be 'medium'). if rng.random() < 0.5 and o["size"] != "medium": q = f"is the {o['color']} {o['shape']} big or small" else: q = _pick(rng, [f"how big is the {o['color']} {o['shape']}", f"what size is the {o['color']} {o['shape']}", f"what size is the {o['color']} {o['shape']}"]) return q, SIZE_WORD[o["size"]][0] if qt == "caption": parts = [] for sh in sorted({o["shape"] for o in obs}): for col in sorted({o["color"] for o in obs}): k = len(_distinct_shape(scene, shape=sh, color=col)) if k: parts.append(f"{NUM_WORD[k]} {col} {_plural(sh, k)}") ans = " and ".join(parts) return _pick(rng, ["describe the image", "what do you see", "what is in the picture"]), ans if qt == "total": q = _pick(rng, ["how many objects are there", "how many items are in the image", "count everything in the picture"]) return q, NUM_WORD[len(obs)] if qt == "compare": col = _pick(rng, sorted({o["color"] for o in obs})) sh = _pick(rng, sorted({o["shape"] for o in obs})) if col == sh: continue a = len(_distinct_shape(scene, color=col)) b = len(_distinct_shape(scene, shape=sh)) if a == b: continue q = _pick(rng, [f"are there more {col} objects than {_plural(sh, 2)}", f"do we see more {col} things than {_plural(sh, 2)}"]) return q, ("yes" if a > b else "no") if qt == "presence": if rng.random() < 0.5: o = _pick(rng, sos) q = _pick(rng, [f"is there a {o['color']} {o['shape']}", f"do you see a {o['color']} {o['shape']}"]) return q, "yes" col = _pick(rng, list(COLORS)) sh = _pick(rng, SHAPES) if _distinct_shape(scene, color=col, shape=sh): continue q = _pick(rng, [f"is there a {col} {sh}", f"do you see a {col} {sh}"]) return q, "no" if qt == "relation": if len(obs) < 2: continue anchor = sorted(obs, key=lambda o: o["cx"])[-1] cands = [o for o in obs if o is not anchor and o["cx"] < anchor["cx"] - 0.05] rel = _pick(rng, ["left", "above", "below"]) if rel == "left": cands = [o for o in obs if o is not anchor and o["cx"] < anchor["cx"] - 0.05] q = f"what color is the object left of the {anchor['color']} {anchor['shape']}" elif rel == "above": cands = [o for o in obs if o is not anchor and o["cy"] < anchor["cy"] - 0.05] q = f"what color is the object above the {anchor['color']} {anchor['shape']}" else: cands = [o for o in obs if o is not anchor and o["cy"] > anchor["cy"] + 0.05] q = f"what color is the object below the {anchor['color']} {anchor['shape']}" if not cands: continue target = min(cands, key=lambda o: abs(o["cx"] - anchor["cx"]) if rel == "left" else abs(o["cy"] - anchor["cy"])) return q, target["color"] except (IndexError, ValueError): continue o = obs[0] return f"what color is the {o['shape']}", o["color"] def render_and_ask(seed: int, index: int, px: int = 128, qtypes=None): """Deterministic (image, question, answer, scene) for global sample `index`.""" rng = np.random.default_rng((seed * 1000003 + index) % (2 ** 63)) scene = make_scene(rng) q, a = question_answer(scene, rng, qtypes) img = render_scene(scene, px=px) return img, q, a, scene class SceneDataset: """torch-free dataset; `fetch(batch_indices)` is what the trainer calls. Determinism note: sample content depends only on (seed, index), never on batch order, worker id or RNG draws. That is what makes a resumed run continue *exactly* where it stopped. """ def __init__(self, seed: int = 0, px: int = 128, qtypes=None): self.seed, self.px, self.qtypes = seed, px, qtypes def __len__(self): return 2 ** 40 def __getitem__(self, index: int): return render_and_ask(self.seed, int(index), self.px, self.qtypes) def fetch(self, indices, workers: int = 8): if workers <= 1: return [self[i] for i in indices] import concurrent.futures as cf with cf.ThreadPoolExecutor(max_workers=workers) as ex: return list(ex.map(self.__getitem__, indices)) # -------------------------------------------------------------------------- # Batch spec generation (cheap, CPU) - the heavy rasterisation moves to the # GPU (see omni/render.py) so the trainer is not starved by the CPU. # -------------------------------------------------------------------------- SHAPE_ID = {s_: i for i, s_ in enumerate(SHAPES)} COLOR_NAMES = list(COLORS) SIZE_ID = {k: i for i, k in enumerate(SIZE_R)} def spec_batch(indices, seed: int = 0, max_obj: int = 4, px: int = 128): """Return padded numpy arrays describing a batch of scenes + their Q/A.""" import numpy as np B = len(indices) shape = np.zeros((B, max_obj), np.int64) color = np.zeros((B, max_obj, 3), np.float32) cx = np.zeros((B, max_obj), np.float32) cy = np.zeros((B, max_obj), np.float32) rad = np.zeros((B, max_obj), np.float32) valid = np.zeros((B, max_obj), np.float32) areas = np.zeros((B, max_obj), np.float32) bg = np.zeros((B, 3), np.float32) scenes, questions, answers = [], [], [] for b, idx in enumerate(indices): rng = np.random.default_rng((seed * 1000003 + int(idx)) % (2 ** 63)) sc = make_scene(rng) q, a = question_answer(sc, rng) scenes.append(sc); questions.append(q); answers.append(a) bg[b] = np.array(BG[sc["bg"]], np.float32) / 255.0 for k, ob in enumerate(sc["objects"][:max_obj]): shape[b, k] = SHAPE_ID[ob["shape"]] color[b, k] = np.array(COLORS[ob["color"]], np.float32) / 255.0 cx[b, k], cy[b, k] = ob["cx"], ob["cy"] rad[b, k] = SIZE_R[ob["size"]] / px * (1.0 + 0.12 * ob.get("jitter", 0.0)) valid[b, k] = 1.0 areas[b, k] = np.pi * rad[b, k] ** 2 return {"shape": shape, "color": color, "cx": cx, "cy": cy, "r": rad, "valid": valid, "areas": areas, "bg": bg, "questions": questions, "answers": answers, "scenes": scenes} return @app.cell(hide_code=True) def lib_render(S): # ===== omni/render.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/render.py using only the imports this module actually needs. def _smooth(x): return x.clamp(0.0, 1.0) def render_specs(spec: dict, px: int = 128, device="cuda", ss: int = 2): """spec: output of scenes.spec_batch -> (B, 3, px, px) float in [0, 1].""" B, K = spec["shape"].shape shape = torch.as_tensor(spec["shape"], device=device) color = torch.as_tensor(spec["color"], device=device) cx = torch.as_tensor(spec["cx"], device=device) cy = torch.as_tensor(spec["cy"], device=device) rad = torch.as_tensor(spec["r"], device=device) valid = torch.as_tensor(spec["valid"], device=device) bg = torch.as_tensor(spec["bg"], device=device) n = px * ss t = (torch.arange(n, device=device, dtype=torch.float32) + 0.5) / n yy = t[None, :, None].expand(B, n, n) xx = t[None, None, :].expand(B, n, n) img = bg[:, :, None, None].expand(B, 3, n, n).contiguous() for k in range(K): v = valid[:, k].view(B, 1, 1) if float(v.sum()) == 0: continue r = rad[:, k].view(B, 1, 1) dx = xx - cx[:, k].view(B, 1, 1) dy = yy - cy[:, k].view(B, 1, 1) d = torch.sqrt(dx * dx + dy * dy + 1e-8) sid = shape[:, k] cov = torch.zeros_like(dx) for sid_, name in enumerate(S.SHAPES): sel = (sid == sid_).view(B, 1, 1) if not bool(sel.any()): continue if name == "circle": sd = r - d elif name == "square": sd = r - torch.maximum(dx.abs(), dy.abs()) elif name == "diamond": sd = r * 1.25 - (dx.abs() + dy.abs()) elif name == "triangle": half = r * ((dy + r) / (2 * r)).clamp(0, 1) sd = torch.minimum(half - dx.abs(), torch.minimum(r - dy, dy + r)) elif name == "hexagon": ang = torch.atan2(dy, dx) rr = r * (0.8660 / torch.cos(((ang + 3.14159265) % 1.0471975) - 0.5235988).clamp(min=1e-3)) sd = rr - d else: # star ang = torch.atan2(dy, dx) rr = r * (1.0 - 0.45 * torch.cos(5 * ang)) sd = rr - d cov = torch.where(sel, _smooth(sd * (px * ss) / max(1.0, 2.0) + 0.5), cov) cov = cov * v col = color[:, k, :, None, None] # cheap directional shading, same as the numpy reference renderer shade = (1.0 + 0.18 * (yy - cy[:, k].view(B, 1, 1)) / r.clamp(min=1e-3)).clamp(0.6, 1.4) ink = (col * shade[:, None]).clamp(0, 1) img = img * (1 - cov[:, None]) + ink * cov[:, None] if ss > 1: img = F.avg_pool2d(img, ss) return img.clamp(0, 1) return @app.cell(hide_code=True) def lib_vlm(): # ===== omni/vlm.py ===== module body # Imports live in `setup`; `materialize` writes this back out to # /marimo/omni/vlm.py so detached trainers and future sandboxes import # exactly the code that lives in this notebook. @dataclass class VLMConfig: d_model: int = 512 n_layers: int = 8 n_heads: int = 8 mlp_ratio: float = 4.0 dropout: float = 0.0 img_size: int = 128 patch: int = 8 vision_d_model: int = 320 vision_layers: int = 6 vision_heads: int = 5 n_mels: int = 80 n_phones: int = 44 frame_pos_max: int = 768 @property def n_patches(self): return (self.img_size // self.patch) ** 2 def to_dict(self): return asdict(self) @classmethod def from_dict(cls, d): return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__}) class CombinedVocab: """One id space for everything the trunk reads and writes.""" SPECIAL_WORDS = ["", "", "", "", "", "", ""] def __init__(self, words, phones): self.words = list(self.SPECIAL_WORDS) + [w for w in words if w not in self.SPECIAL_WORDS] self.phones = list(phones) self.w2i = {w: i for i, w in enumerate(self.words)} self.p2i = {p: len(self.words) + i for i, p in enumerate(self.phones)} self.img, self.speak, self.sep = self.w2i[""], self.w2i[""], self.w2i[""] self.pad, self.bos, self.eos = self.w2i[""], self.w2i[""], self.w2i[""] self.mask = self.w2i[""] self.n_words, self.n_phones = len(self.words), len(self.phones) def __len__(self): return self.n_words + self.n_phones def words_to_ids(self, ws): return [self.w2i.get(w, self.mask) for w in ws] def phones_to_ids(self, ps): return [self.p2i.get(p, self.p2i[self.phones[0]]) for p in ps] def ids_to_words(self, ids): out = [] for i in ids: i = int(i) if 0 <= i < self.n_words: w = self.words[i] if w in ("", ""): break if not w.startswith("<"): out.append(w) return out def ids_to_phones(self, ids): out = [] for i in ids: i = int(i) if self.n_words <= i < len(self): out.append(self.phones[i - self.n_words]) return out # --------------------------------------------------------------------- RoPE -- def rope_tables(dim, seq_len, device, base=10000.0): inv = 1.0 / (base ** (torch.arange(0, dim, 2, device=device).float() / dim)) t = torch.arange(seq_len, device=device).float() f = torch.outer(t, inv) return torch.cos(f), torch.sin(f) def apply_rope(x, cos, sin): x1, x2 = x[..., 0::2], x[..., 1::2] c, s = cos[None, None], sin[None, None] return torch.stack([x1 * c - x2 * s, x1 * s + x2 * c], dim=-1).flatten(-2) class SelfAttention(nn.Module): def __init__(self, d, n_heads, dropout=0.0): super().__init__() assert d % n_heads == 0 self.h, self.dh = n_heads, d // n_heads self.qkv = nn.Linear(d, 3 * d, bias=False) self.proj = nn.Linear(d, d, bias=False) self.drop = dropout def forward(self, x, cache=None, cos=None, sin=None): B, T, D = x.shape qkv = self.qkv(x).view(B, T, 3, self.h, self.dh).permute(2, 0, 3, 1, 4) q, k, v = qkv[0], qkv[1], qkv[2] if cos is not None: q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin) if cache is not None: prefill = cache.k is None # first pass over a prefix: still causal! k, v = cache.update(k, v) causal = prefill else: causal = True o = F.scaled_dot_product_attention(q, k, v, is_causal=causal, dropout_p=self.drop if self.training else 0.0) return self.proj(o.transpose(1, 2).reshape(B, T, D)) class CrossAttention(nn.Module): """Attends a fixed memory (vision tokens, or a learned null memory).""" def __init__(self, d, n_heads): super().__init__() self.h, self.dh = n_heads, d // n_heads self.q = nn.Linear(d, d, bias=False) self.kv = nn.Linear(d, 2 * d, bias=False) self.proj = nn.Linear(d, d, bias=False) def forward(self, x, mem, mem_cache=None): B, T, D = x.shape q = self.q(x).view(B, T, self.h, self.dh).transpose(1, 2) if mem_cache is not None and mem_cache[0] is not None: k, v = mem_cache else: k, v = self.kv(mem).view(B, mem.shape[1], 2, self.h, self.dh).permute(2, 0, 3, 1, 4) if mem_cache is not None: mem_cache[0], mem_cache[1] = k, v o = F.scaled_dot_product_attention(q, k, v) return self.proj(o.transpose(1, 2).reshape(B, T, D)) class KVCache: def __init__(self): self.k = self.v = None @property def pos(self): return 0 if self.k is None else self.k.shape[2] def update(self, k, v): self.k, self.v = (k, v) if self.k is None else (torch.cat([self.k, k], 2), torch.cat([self.v, v], 2)) return self.k, self.v class Block(nn.Module): def __init__(self, cfg, with_cross=True): super().__init__() d = cfg.d_model self.ln1 = nn.LayerNorm(d) self.self_attn = SelfAttention(d, cfg.n_heads, cfg.dropout) self.with_cross = with_cross if with_cross: self.ln2 = nn.LayerNorm(d) self.cross = CrossAttention(d, cfg.n_heads) self.ln3 = nn.LayerNorm(d) self.mlp = nn.Sequential(nn.Linear(d, int(d * cfg.mlp_ratio)), nn.GELU(), nn.Linear(int(d * cfg.mlp_ratio), d)) def forward(self, x, mem=None, cache=None, cos=None, sin=None, mem_cache=None): x = x + self.self_attn(self.ln1(x), cache=cache, cos=cos, sin=sin) if self.with_cross and mem is not None: x = x + self.cross(self.ln2(x), mem, mem_cache=mem_cache) return x + self.mlp(self.ln3(x)) class ViT(nn.Module): """Vision encoder, trained from scratch on rendered scenes.""" def __init__(self, cfg: VLMConfig, in_ch=3): super().__init__() d = cfg.vision_d_model self.cfg = cfg self.patch = nn.Conv2d(in_ch, d, cfg.patch, cfg.patch) self.cls = nn.Parameter(torch.zeros(1, 1, d)) self.pos = nn.Parameter(torch.zeros(1, cfg.n_patches + 1, d)) nn.init.trunc_normal_(self.pos, std=0.02) nn.init.trunc_normal_(self.cls, std=0.02) vcfg = replace(cfg, d_model=cfg.vision_d_model, n_heads=cfg.vision_heads) self.blocks = nn.ModuleList([Block(vcfg, with_cross=False) for _ in range(cfg.vision_layers)]) self.ln = nn.LayerNorm(d) self.proj = nn.Linear(d, cfg.d_model, bias=False) def forward(self, images): z = self.patch(images).flatten(2).transpose(1, 2) z = torch.cat([self.cls.expand(z.shape[0], -1, -1), z], 1) + self.pos for b in self.blocks: z = b(z) return self.proj(self.ln(z)) class MelDecoder(nn.Module): """FastSpeech-style frame decoder: per-frame hidden -> mel.""" def __init__(self, cfg: VLMConfig): super().__init__() d = cfg.d_model self.inp = nn.Linear(d, d) self.frame_pos = nn.Parameter(torch.zeros(1, cfg.frame_pos_max, d)) nn.init.trunc_normal_(self.frame_pos, std=0.02) self.convs = nn.ModuleList([nn.Conv1d(d, d, 5, padding=2 * k, dilation=k) for k in (1, 2, 4, 8)]) self.norms = nn.ModuleList([nn.LayerNorm(d) for _ in range(4)]) self.out = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, d), nn.GELU(), nn.Linear(d, cfg.n_mels)) def forward(self, h): B, T, D = h.shape idx = torch.arange(T, device=h.device) % self.frame_pos.shape[1] x = self.inp(h) + self.frame_pos[:, idx] for c, n in zip(self.convs, self.norms): y = c(x.transpose(1, 2)).transpose(1, 2) x = x + F.gelu(n(y)) return self.out(x) class OmniVLM(nn.Module): def __init__(self, cfg: VLMConfig, n_words: int, n_phones: int): super().__init__() self.cfg = cfg self.n_words, self.n_phones = n_words, n_phones self.tok = nn.Embedding(n_words + n_phones, cfg.d_model) self.vision = ViT(cfg) self.null_mem = nn.Parameter(torch.zeros(1, 1, cfg.d_model)) nn.init.trunc_normal_(self.null_mem, std=0.02) self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layers)]) self.ln_f = nn.LayerNorm(cfg.d_model) self.head_text = nn.Linear(cfg.d_model, n_words, bias=False) self.head_phone = nn.Linear(cfg.d_model, n_phones, bias=False) self.head_dur = nn.Sequential(nn.Linear(cfg.d_model, 256), nn.GELU(), nn.Linear(256, 1)) self.mel_dec = MelDecoder(cfg) self.apply(self._init) @staticmethod def _init(m): if isinstance(m, nn.Linear): nn.init.trunc_normal_(m.weight, std=0.02) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, nn.Embedding): nn.init.trunc_normal_(m.weight, std=0.02) elif isinstance(m, nn.LayerNorm): nn.init.ones_(m.weight) nn.init.zeros_(m.bias) # ------------------------------------------------------------- primitives def trunk(self, ids, mem=None, cache=None, mem_cache=None, start=0): B, T = ids.shape x = self.tok(ids) if mem is None: mem = self.null_mem.expand(B, 1, -1) # always cross-attend something dh = self.cfg.d_model // self.cfg.n_heads cos, sin = rope_tables(dh, start + T, x.device) cos, sin = cos[start:start + T], sin[start:start + T] for i, blk in enumerate(self.blocks): x = blk(x, mem=mem, cache=None if cache is None else cache[i], cos=cos, sin=sin, mem_cache=mem_cache if mem_cache is None else mem_cache[i]) return self.ln_f(x) # ------------------------------------------------------------ mode: text def forward_text(self, ids, images=None, targets=None, loss_mask=None, mem=None): if mem is None and images is not None: mem = self.vision(images) h = self.trunk(ids, mem=mem) logits = self.head_text(h) out = {"logits": logits, "hidden": h} if targets is not None: m = torch.ones_like(targets, dtype=torch.bool) if loss_mask is None else loss_mask.bool() m = m & (targets >= 0) & (targets < self.n_words) safe = torch.where(m, targets, torch.zeros_like(targets)) ls = F.cross_entropy(logits.reshape(-1, logits.shape[-1]), safe.reshape(-1), reduction="none") out["loss"] = (ls * m.reshape(-1)).sum() / m.sum().clamp(min=1) with torch.no_grad(): acc = ((logits.argmax(-1) == safe) & m).sum() / m.sum().clamp(min=1) out["acc"] = acc return out @torch.no_grad() def generate_text(self, ids, images=None, max_new=24, temperature=0.0, top_k=0, eos=None, mem=None, return_probs=False): """KV-cached autoregressive decode: the fast-generation path.""" self.eval() if mem is None and images is not None: mem = self.vision(images) cache = [KVCache() for _ in self.blocks] mem_cache = [[None, None] for _ in self.blocks] h = self.trunk(ids, mem=mem, cache=cache, mem_cache=mem_cache) logits = self.head_text(h[:, -1]) beos = self.eos_id if eos is None else eos outs, probs = [], [] for _ in range(max_new): nxt = self._sample(logits, temperature, top_k) outs.append(nxt) if return_probs: probs.append(F.softmax(logits.float(), -1)) if bool((nxt == beos).all()): break h = self.trunk(nxt[:, None], cache=cache, mem_cache=mem_cache, start=cache[0].pos) logits = self.head_text(h[:, -1]) ids_out = torch.stack(outs, 1) if outs else torch.zeros(ids.shape[0], 0, dtype=torch.long, device=ids.device) return (ids_out, torch.stack(probs, 1)) if return_probs else ids_out @staticmethod def _sample(logits, temperature=0.0, top_k=0): if temperature <= 0: return logits.argmax(-1) logits = logits / temperature if top_k: v = logits.topk(top_k, dim=-1).values[..., -1:] logits = logits.masked_fill(logits < v, -float("inf")) return torch.multinomial(F.softmax(logits, -1), 1).squeeze(-1) # ---------------------------------------------------------- mode: speech def forward_speech(self, text_ids, phone_ids, assign, mel=None, mask=None, durations=None, mem=None): """Teacher-forced speech pass. text_ids : (B, Tt) transcript words phone_ids : (B, Tp) ARPAbet phones for that transcript (from G2P) assign : (B, Tm) index of the phone each *mel frame* belongs to durations : (B, Tp) mel frames per phone (== assign, summarised) mel : (B, Tm, n_mels) target log-mel """ B = text_ids.shape[0] sep = torch.full((B, 1), self.sep_id, dtype=torch.long, device=text_ids.device) start = torch.full((B, 1), self.pad_id, dtype=torch.long, device=text_ids.device) phone_in = torch.cat([start, phone_ids[:, :-1]], 1) h = self.trunk(torch.cat([text_ids, sep, phone_in], 1), mem=mem) hp = h[:, text_ids.shape[1] + 1:] out = {"phone_logits": self.head_phone(hp), "dur_pred": self.head_dur(hp).squeeze(-1)} lm = phone_ids >= 0 # phone_ids arrive in the *shared* id space (n_words + phone index); the # phone head predicts the local phone index, so shift before the loss. tgt_phone = (phone_ids - self.n_words).clamp(min=0) pl = F.cross_entropy(out["phone_logits"].reshape(-1, self.n_phones), tgt_phone.reshape(-1), reduction="none") out["phone_loss"] = (pl * lm.reshape(-1)).sum() / lm.reshape(-1).sum().clamp(min=1) with torch.no_grad(): out["phone_acc"] = (((out["phone_logits"].argmax(-1) == tgt_phone) & lm).sum() / lm.sum().clamp(min=1)) if durations is not None: tgt = torch.log1p(durations.float().clamp(0, 400)) dp = F.smooth_l1_loss(out["dur_pred"].clamp(0, 6), tgt, reduction="none") out["dur_loss"] = (dp * lm).sum() / lm.sum().clamp(min=1) if mel is not None: Tm = min(mel.shape[1], assign.shape[1]) idx = assign[:, :Tm].clamp(0, hp.shape[1] - 1) frames = torch.gather(hp, 1, idx[..., None].expand(-1, -1, hp.shape[-1])) pred = self.mel_dec(frames) mm = (torch.ones(pred.shape[:2], dtype=torch.bool, device=pred.device) if mask is None else mask[:, :Tm]) out["mel_pred"] = pred out["mel_loss"] = (F.l1_loss(pred, mel[:, :Tm], reduction="none").mean(-1) * mm).sum() / mm.sum().clamp(min=1) return out @staticmethod def expand_to_frames(h_phones, durations): d = durations.clamp(min=0).round().long() outs = [] for b in range(h_phones.shape[0]): idx = torch.repeat_interleave(torch.arange(h_phones.shape[1], device=h_phones.device), d[b]) outs.append(h_phones[b][idx] if idx.numel() else h_phones[b][:0]) T = max(o.shape[0] for o in outs) if T == 0: return h_phones.new_zeros(h_phones.shape[0], 0, h_phones.shape[-1]) padded = h_phones.new_zeros(h_phones.shape[0], T, h_phones.shape[-1]) for b, o in enumerate(outs): padded[b, :o.shape[0]] = o return padded @torch.no_grad() def synth(self, text_ids, max_phones=400, temperature=0.0, eos=None): """text -> (phones, durations, mel). The voice path, KV-cached.""" self.eval() B = text_ids.shape[0] sep = torch.full((B, 1), self.sep_id, dtype=torch.long, device=text_ids.device) cache = [KVCache() for _ in self.blocks] mem_cache = [[None, None] for _ in self.blocks] h = self.trunk(torch.cat([text_ids, sep], 1), cache=cache, mem_cache=mem_cache) hs = h[:, -1] phones, durs, hidden = [], [], [] for _ in range(max_phones): ph_local = self._sample(self.head_phone(hs), temperature, 0) ph = ph_local + self.n_words # back into the shared id space du = torch.expm1(self.head_dur(hs).squeeze(-1).clamp(0, 6)).round().clamp(1, 60) phones.append(ph) durs.append(du) hidden.append(hs) hs = self.trunk(ph[:, None], cache=cache, mem_cache=mem_cache, start=cache[0].pos)[:, -1] phone_ids = torch.stack(phones, 1) dur = torch.stack(durs, 1).long() H = torch.stack(hidden, 1) frames = self.expand_to_frames(H, dur) mel = (self.mel_dec(frames) if frames.shape[1] else H.new_zeros(H.shape[0], 0, self.cfg.n_mels)) # `hidden` is returned so a caller can re-time the timeline (speaking # rate calibration) and rebuild `mel` with the *same* durations that the # lips will use - otherwise audio and lips drift apart. return {"phones": phone_ids, "durations": dur, "mel": mel, "hidden": H} @torch.no_grad() def mel_from_durations(self, hidden, durations): """Rebuild the mel for an explicit timeline (used by rate calibration).""" frames = self.expand_to_frames(hidden, durations) if not frames.shape[1]: return hidden.new_zeros(hidden.shape[0], 0, self.cfg.n_mels) return self.mel_dec(frames) pad_id, sep_id, eos_id, bos_id, mask_id = 3, 2, 5, 4, 6 def count_params(m: nn.Module) -> int: return sum(p.numel() for p in m.parameters()) return CombinedVocab, OmniVLM, VLMConfig, count_params @app.cell(hide_code=True) def lib_speech(textmod): # ===== omni/speech.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/speech.py using only the imports this module actually needs. SR = 22050 N_FFT = 1024 HOP = 256 WIN = 1024 N_MELS = 80 FMIN, FMAX = 0.0, 8000.0 # ------------------------------------------------------------------ frontend def mel_filterbank(sr=SR, n_fft=N_FFT, n_mels=N_MELS, fmin=FMIN, fmax=FMAX): def hz2mel(f): return 2595.0 * np.log10(1.0 + f / 700.0) def mel2hz(m): return 700.0 * (10 ** (m / 2595.0) - 1.0) mels = np.linspace(hz2mel(fmin), hz2mel(fmax), n_mels + 2) freqs = np.linspace(0, sr / 2, n_fft // 2 + 1) fb = np.zeros((n_mels, len(freqs)), np.float32) for i in range(n_mels): lo, mid, hi = mel2hz(mels[i]), mel2hz(mels[i + 1]), mel2hz(mels[i + 2]) up = (freqs - lo) / max(mid - lo, 1e-6) down = (hi - freqs) / max(hi - mid, 1e-6) fb[i] = np.clip(np.minimum(up, down), 0, None) fb /= np.maximum(fb.sum(1, keepdims=True), 1e-8) return fb _MEL_FB: dict[tuple, torch.Tensor] = {} _WINDOW: dict[tuple, torch.Tensor] = {} def _window(device, win: int = WIN): """Window cache keyed by (length, device) - never a single global.""" key = (win, str(device)) w = _WINDOW.get(key) if w is None: w = torch.hann_window(win, device=device) _WINDOW[key] = w return w def mel_spectrogram(wav: torch.Tensor, sr=SR, n_mels: int | None = None, n_fft: int = N_FFT, hop: int = HOP, win: int = WIN) -> torch.Tensor: """(B, T) waveform in [-1, 1] -> (B, n_mels, frames) log-mel. The filterbank geometry is a parameter, not a global: a model trained with a different ``n_mels`` must not silently receive an 80-bin spectrogram. """ n_mels = int(n_mels or N_MELS) key = (n_mels, n_fft, str(wav.device)) fb = _MEL_FB.get(key) if fb is None: fb = torch.from_numpy(mel_filterbank(sr=sr, n_fft=n_fft, n_mels=n_mels)).to(wav.device) _MEL_FB[key] = fb x = torch.stft(wav.float(), n_fft, hop, win, window=_window(wav.device, win), center=True, return_complex=True, pad_mode="reflect") return torch.log(torch.matmul(fb, x.abs() ** 2).clamp(min=1e-5)) def load_wav(path, sr=SR) -> np.ndarray: from scipy.io import wavfile rate, data = wavfile.read(str(path)) x = np.asarray(data) if x.dtype == np.int16: x = x.astype(np.float32) / 32768.0 elif x.dtype == np.int32: x = x.astype(np.float32) / 2147483648.0 else: x = x.astype(np.float32) if x.ndim > 1: x = x.mean(1) return x.astype(np.float32) def write_wav(path, x: np.ndarray, sr=SR): from scipy.io import wavfile x = np.clip(np.asarray(x, np.float32), -1, 1) wavfile.write(str(path), sr, (x * 32767).astype(np.int16)) # ----------------------------------------------------------------- LJSpeech class LJSpeech: """Public-domain LibriVox corpus: 13,100 clips, ~24 h, one speaker.""" def __init__(self, root="/marimo/data/LJSpeech-1.1"): self.root = Path(root) meta = self.root / "metadata.csv" self.items = [] for line in meta.read_text(errors="replace").splitlines(): parts = line.split("|") if len(parts) < 3: continue self.items.append({"id": parts[0], "text": parts[2], "raw": parts[1]}) self.by_text = {it["text"]: it for it in self.items} def __len__(self): return len(self.items) def wav_path(self, i): return self.root / "wavs" / f"{self.items[i]['id']}.wav" def load(self, i): return load_wav(self.wav_path(i)), self.items[i] def prepared(self): return [it for it in self.items if Path(self.root / "wavs" / f"{it['id']}.wav").exists()] # ----------------------------------------------------------------------- Ear class EarVocab: def __init__(self): self.itos = [""] + textmod.PHONES self.stoi = {p: i for i, p in enumerate(self.itos)} def __len__(self): return len(self.itos) def encode(self, phones): return [self.stoi.get(p, 0) for p in phones] class EarBlock(nn.Module): def __init__(self, d, n_heads, ff=4): super().__init__() self.ln1, self.ln2 = nn.LayerNorm(d), nn.LayerNorm(d) self.attn = nn.MultiheadAttention(d, n_heads, batch_first=True) self.ff = nn.Sequential(nn.Linear(d, d * ff), nn.GELU(), nn.Linear(d * ff, d)) def forward(self, x, mask): h = self.ln1(x) a, _ = self.attn(h, h, h, key_padding_mask=mask, need_weights=False) x = x + a return x + self.ff(self.ln2(x)) class Ear(nn.Module): """CTC acoustic model: mel frames -> phoneme sequence (and alignment).""" def __init__(self, n_mels=N_MELS, d=256, n_layers=4, n_heads=4, n_out=40, stride=2): super().__init__() self.stem = nn.Sequential( nn.Conv1d(n_mels, d, 5, padding=2), nn.GELU(), nn.GroupNorm(1, d), nn.Conv1d(d, d, 5, stride=stride, padding=2), nn.GELU(), nn.GroupNorm(1, d), nn.Conv1d(d, d, 5, stride=stride, padding=2), nn.GELU(), nn.GroupNorm(1, d)) self.blocks = nn.ModuleList([EarBlock(d, n_heads) for _ in range(n_layers)]) self.out = nn.Linear(d, n_out) def forward(self, mel): # mel: (B, n_mels, T) x = self.stem(mel).transpose(1, 2) # (B, T/4, d) T = x.shape[1] x = x + _sinusoid(T, x.shape[-1], x.device)[None] for b in self.blocks: x = b(x, None) return F.log_softmax(self.out(x), -1) # (B, T/4, n_out) def _sinusoid(T, d, device): pos = torch.arange(T, device=device).float()[:, None] div = torch.exp(torch.arange(0, d, 2, device=device).float() * (-math.log(10000.0) / d)) pe = torch.zeros(T, d, device=device) pe[:, 0::2] = torch.sin(pos * div) pe[:, 1::2] = torch.cos(pos * div) return pe def ear_token_frames(stride=4): return stride # --------------------------------------------------------------- CTC helpers def ctc_viterbi_align(logp, targets, blank: int = 0, upsample: int = 4, n_mel: int | None = None): """Forced alignment of a known target sequence to a CTC posterior. Only the time axis is looped; the state axis is vectorised, which turns a ~1 s/clip Python Viterbi into ~10 ms/clip for the whole corpus. Returns (frame_to_phone, durations) in *mel* frames, so that sum(durations) == number of mel frames and each mel frame knows which phone produced it (blanks are absorbed into the neighbouring phone). """ lp = np.asarray(logp.detach().float().cpu().numpy() if hasattr(logp, "detach") else logp, np.float32) T = lp.shape[0] S = 2 * len(targets) + 1 ext = np.full(S, blank, np.int64) for i, t in enumerate(targets): ext[2 * i + 1] = t skip_ok = np.zeros(S, bool) skip_ok[2:] = (ext[2:] != ext[:-2]) NEG = np.float32(-1e30) dp = np.full((T + 1, S), NEG, np.float32) bp = np.zeros((T + 1, S), np.int8) dp[0, 0] = 0.0 for t in range(1, T + 1): prev = dp[t - 1] stay = prev one = np.concatenate([np.full(1, NEG, np.float32), prev[:-1]]) two = np.concatenate([np.full(2, NEG, np.float32), prev[:-2]]) two = np.where(skip_ok, two, NEG) stack = np.stack([stay, one, two]) arg = stack.argmax(0).astype(np.int8) dp[t] = stack[arg, np.arange(S)] + lp[t - 1, ext] bp[t] = arg s = int(np.argmax(dp[T])) labels = np.zeros(T, np.int64) #: which phone *of the target sequence* each frame belongs to. A label state in #: the extended sequence sits at position (s - 1) // 2. Tracking the state is #: what keeps a repeated phone's occurrences apart. positions = np.full(T, -1, np.int64) t = T while t > 0: labels[t - 1] = ext[s] positions[t - 1] = (s - 1) // 2 if (s % 2 == 1) else -1 s -= int(bp[t, s]) t -= 1 # absorb blanks into the nearest phone (forward fill, then backward fill), # carrying each phone's position along with its label filled = labels.copy() fpos = positions.copy() for t in range(1, T): if filled[t] == blank and filled[t - 1] != blank: filled[t] = filled[t - 1] fpos[t] = fpos[t - 1] for t in range(T - 2, -1, -1): if filled[t] == blank: if t + 1 < T: filled[t] = filled[t + 1] fpos[t] = fpos[t + 1] else: fpos[t] = 0 if len(targets) == 0: return np.zeros(0, np.int64), np.zeros(0, np.int64) # This used to be `pos = {p: i for i, p in enumerate(targets)}`, which maps a # phone id to the position of its *last* occurrence in the target sequence. In # ordinary speech phones repeat constantly, so every earlier occurrence was # given zero frames while the last one absorbed hundreds: measured over the # corpus, 65% of phones had duration 0 and the maximum was 673 frames. Both # the duration head and the mel decoder learn from this, which is why the # voice and the lips drifted. The position must come from the path, not from # a dictionary keyed on the phone id. phone_pos = np.clip(fpos, 0, len(targets) - 1) # ear frames -> mel frames Tm = n_mel if n_mel is not None else T * upsample src = np.clip(np.arange(Tm) // upsample, 0, max(1, T) - 1) assign = phone_pos[src] # any leading unassigned frame -> first phone if (assign < 0).any(): first = int(assign[assign >= 0][0]) if (assign >= 0).any() else 0 assign[assign < 0] = first dur = np.bincount(assign, minlength=len(targets)).astype(np.int64) return assign, dur # ------------------------------------------------------------------- vocoder class ConvNeXtBlock(nn.Module): def __init__(self, d, k=7): super().__init__() self.dw = nn.Conv1d(d, d, k, padding=k // 2, groups=d) self.norm = nn.LayerNorm(d) self.pw1 = nn.Linear(d, 4 * d) self.pw2 = nn.Linear(4 * d, d) def forward(self, x): h = self.dw(x).transpose(1, 2) h = self.pw2(F.gelu(self.pw1(self.norm(h)))) return x + h.transpose(1, 2) class Vocoder(nn.Module): """Vocos-style: log-mel -> complex STFT -> istft. Trained from scratch.""" def __init__(self, n_mels=N_MELS, d=256, n_layers=6, n_fft=N_FFT, hop=HOP, win=WIN): super().__init__() self.n_fft, self.hop, self.win = n_fft, hop, win self.inp = nn.Conv1d(n_mels, d, 7, padding=3) self.blocks = nn.ModuleList([ConvNeXtBlock(d) for _ in range(n_layers)]) self.norm = nn.LayerNorm(d) self.head = nn.Linear(d, 2 * (n_fft // 2 + 1)) self.register_buffer("window", torch.hann_window(win)) self.register_buffer("freqs", torch.arange(n_fft // 2 + 1).float()) def forward(self, mel): x = self.inp(mel) for b in self.blocks: x = b(x) h = self.norm(x.transpose(1, 2)) s = self.head(h).float() # (B, T, 2*F) Fb = self.n_fft // 2 + 1 mag = torch.exp(s[..., :Fb].clamp(-12, 6)) phase = torch.sin(s[..., Fb:]) * math.pi spec = torch.polar(mag, phase) wav = torch.istft(spec.transpose(1, 2), self.n_fft, self.hop, self.win, window=self.window, center=True, length=mel.shape[-1] * self.hop) return wav def stft_loss(x, y, ffts=((512, 128, 512), (1024, 256, 1024), (2048, 512, 2048))): """Multi-resolution spectral convergence + log-magnitude L1.""" n = min(x.shape[-1], y.shape[-1]) x, y = x[..., :n], y[..., :n] total = 0.0 for n_fft, hop, win in ffts: w = torch.hann_window(win, device=x.device) X = torch.stft(x, n_fft, hop, win, window=w, center=True, return_complex=True).abs() Y = torch.stft(y, n_fft, hop, win, window=w, center=True, return_complex=True).abs() sc = torch.linalg.norm(X - Y, dim=(1, 2)) / (torch.linalg.norm(Y, dim=(1, 2)) + 1e-7) mag = F.l1_loss(torch.log(X + 1e-5), torch.log(Y + 1e-5)) total = total + sc.mean() + mag return total / len(ffts) class MultiResolutionDiscriminator(nn.Module): """HiFi-GAN-style discriminators: the part a GAN-free vocoder is missing. A pure reconstruction loss (multi-resolution STFT) produces clean but *flat* audio: it has no pressure to be sharp or natural. The adversarial term adds that pressure, and the feature-matching term keeps training stable. """ class Sub(nn.Module): def __init__(self, n_fft, hop, win): super().__init__() self.n_fft, self.hop, self.win = n_fft, hop, win self.register_buffer("window", torch.hann_window(win)) def forward(self, x): spec = torch.stft(x, self.n_fft, self.hop, self.win, window=self.window, center=True, return_complex=True) return torch.cat([spec.real, spec.imag], dim=1) # (B, 2F, T) def __init__(self, resolutions=((512, 128), (1024, 256), (2048, 512)), d=64): super().__init__() self.subs = nn.ModuleList([self.Sub(n_fft, hop, n_fft) for n_fft, hop in resolutions]) self.heads = nn.ModuleList() self.feat = nn.ModuleList() for n_fft, hop in resolutions: layers, cin = [], 2 * (n_fft // 2 + 1) for c, k in ((d, 5), (d * 2, 5), (d * 2, 5)): layers += [nn.Conv2d(cin, c, k, stride=(1, 2), padding=k // 2), nn.LeakyReLU(0.1)] self.feat.append(layers[-2]) cin = c self.heads.append(nn.Sequential(*layers, nn.Conv2d(cin, 1, 3, padding=1))) def forward(self, wav): outs, feats = [], [] for sub, head in zip(self.subs, self.heads): x = sub(wav).unsqueeze(2) # (B, 2F, 1, T) h = x for layer in head[:-1]: h = layer(h) feats.append(h) outs.append(head[-1](h)) return outs, feats def discriminator_loss(real_outs, fake_outs): total = 0.0 for r, f in zip(real_outs, fake_outs): total = total + F.relu(1 - r).mean() + F.relu(1 + f).mean() return total / max(1, len(real_outs)) def generator_adv_loss(fake_outs): return sum(-f.mean() for f in fake_outs) / max(1, len(fake_outs)) def feature_match_loss(real_feats, fake_feats): return sum(F.l1_loss(r, f.detach()) for r, f in zip(real_feats, fake_feats)) / max( 1, len(real_feats)) def griffin_lim(mel_log, iters: int = 64, device="cpu", n_mels: int | None = None): """Fallback vocoder so the pipeline runs before the learned one is trained. ``mel_log`` is (n_mels, frames); the bin count comes from the input, so a model with a non-default mel geometry still vocodes. """ mel = torch.as_tensor(np.asarray(mel_log, np.float32), device=device) fb = torch.from_numpy(mel_filterbank(n_mels=int(n_mels or mel.shape[0]))).to(device) power = (torch.linalg.pinv(fb) @ mel).clamp(min=1e-8) mag = power.sqrt() w = torch.hann_window(WIN, device=device) phase = torch.rand_like(mag) * 2 * math.pi wav = torch.zeros(1, device=device) for _ in range(iters): spec = torch.polar(mag, phase) wav = torch.istft(spec.unsqueeze(0), N_FFT, HOP, WIN, window=w, center=True, length=mag.shape[-1] * HOP) rec = torch.stft(wav, N_FFT, HOP, WIN, window=w, center=True, return_complex=True)[0] phase = torch.angle(rec[..., :mag.shape[-1]]) return wav.squeeze(0).cpu().numpy() return @app.cell(hide_code=True) def lib_data(scenes, textmod): # ===== omni/data.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/data.py using only the imports this module actually needs. def ljspeech_words(path="/marimo/data/LJSpeech-1.1/metadata.csv", min_count: int = 2, cap: int = 8000): """Words used by the speech corpus, so one vocabulary serves vision *and* voice: the voxel of the trunk is shared by every modality.""" from collections import Counter c = Counter() p = Path(path) if not p.exists(): return [] for line in p.read_text(errors="replace").splitlines(): parts = line.split("|") if len(parts) >= 3: c.update(textmod.words_of(parts[2])) keep = [(w, n) for w, n in c.items() if n >= min_count] keep.sort(key=lambda kv: (-kv[1], kv[0])) return sorted(w for w, _ in keep[:cap]) def build_vocab(seed: int = 12345, n: int = 4000, extra_words=None, cache="/marimo/runs/shared/vocab_words.json"): """Deterministic word vocabulary: a fixed sample of the scene generator plus the speech corpus vocabulary, so a resumed run on a fresh sandbox rebuilds byte-identical ids.""" cache = Path(cache) if cache.exists(): return json.loads(cache.read_text()) seen = set() for i in range(n): _, q, a, sc = scenes.render_and_ask(seed, i, px=32) seen |= set(textmod.words_of(q)) | set(textmod.words_of(a)) seen |= set(extra_words if extra_words is not None else ljspeech_words()) words = sorted(seen) cache.parent.mkdir(parents=True, exist_ok=True) cache.write_text(json.dumps(words)) return words def vocab_identity(seed: int = 12345, cache="/marimo/runs/shared/vocab_words.json"): """Fingerprint of the token space; part of a run's identity because it changes the model's embedding table.""" import hashlib words = build_vocab(seed, cache=cache) h = hashlib.sha256(json.dumps(words).encode()).hexdigest()[:8] return {"vocab_size": len(words), "vocab_hash": h} def make_vocab(seed: int = 12345, n: int = 4000, cache="/marimo/runs/shared/vocab_words.json"): from omni.vlm import CombinedVocab return CombinedVocab(words=build_vocab(seed, n, cache=cache), phones=textmod.PHONES) def encode_qa(vocab, question: str, answer: str, max_q=32, max_a=32): q = vocab.words_to_ids(textmod.words_of(question))[:max_q] a = vocab.words_to_ids(textmod.words_of(answer))[:max_a] + [vocab.eos] seq = [vocab.bos] + q + [vocab.sep] + a return seq, 1 + len(q) + 1 # loss starts after def collate_text(vocab, samples, device="cuda", img_size=128, max_q=32, max_a=32): """samples: list of (img HWC uint8, question, answer, scene)""" seqs, starts, imgs = [], [], [] for img, q, a, _ in samples: s, st = encode_qa(vocab, q, a, max_q, max_a) seqs.append(s) starts.append(st) imgs.append(img) L = max(len(s) for s in seqs) ids = np.full((len(seqs), L), vocab.pad, np.int64) for i, s in enumerate(seqs): ids[i, :len(s)] = s ids = torch.from_numpy(ids) x, y = ids[:, :-1], ids[:, 1:] pos = torch.arange(y.shape[1])[None, :] mask = pos >= (torch.tensor(starts)[:, None] - 1) img_t = torch.from_numpy(np.stack(imgs).astype(np.float32) / 127.5 - 1.0).permute(0, 3, 1, 2) return {"ids": x.to(device), "targets": y.to(device), "mask": mask.to(device), "images": img_t.to(device)} def collate_specs(vocab, spec, device="cuda", px=128, ss=1, max_q=32, max_a=32): """Spec dict (omni.scenes.spec_batch) -> ready-for-GPU training batch.""" import omni.render as render B = len(spec["questions"]) seqs, starts = [], [] for q, a in zip(spec["questions"], spec["answers"]): s_, st = encode_qa(vocab, q, a, max_q, max_a) seqs.append(s_) starts.append(st) L = max(len(x) for x in seqs) ids = np.full((B, L), vocab.pad, np.int64) for i, x in enumerate(seqs): ids[i, :len(x)] = x ids = torch.from_numpy(ids) x, y = ids[:, :-1], ids[:, 1:] pos = torch.arange(y.shape[1])[None, :] mask = pos >= (torch.tensor(starts)[:, None] - 1) images = render.render_specs(spec, px=px, device=device, ss=ss) return {"ids": x.to(device), "targets": y.to(device), "mask": mask.to(device), "images": images, "questions": spec["questions"], "answers": spec["answers"], "scenes": spec["scenes"], "starts": starts} class Prefetcher: """Pulls index batches from an iterator in a background thread, renders them into GPU-ready specs, and hands them over without ever running dry. The iterator is pure in the step counter, so a resumed run pulls exactly the batches the previous process would have pulled next. """ def __init__(self, fn, index_iter, depth=6): self.fn, self.it = fn, index_iter self.queue = queue.Queue(maxsize=depth) self.stop = False self.consumed = 0 self.thread = threading.Thread(target=self._loop, daemon=True) self.thread.start() def _loop(self): try: for idx in self.it: if self.stop: break self.queue.put(self.fn(idx)) self.consumed += 1 except Exception as e: # pragma: no cover self.queue.put(e) return self.queue.put(None) def __next__(self): b = self.queue.get() if b is None: raise StopIteration if isinstance(b, Exception): raise b return b def __iter__(self): return self def close(self): self.stop = True def batch_indices_for_step(step: int, batch_size: int, seed: int, world: int = 1, rank: int = 0): """Global sample indices for a given optimizer step. A pure function of the step counter => resuming at step N reproduces the exact same batches, with no dependence on where the epoch was interrupted. """ base = (step * batch_size * world) + rank * batch_size rng = np.random.default_rng(seed + step) perm = rng.permutation(batch_size) return (base + perm).tolist() def sample_metric_batches(step: int, n: int, seed: int = 999): rng = np.random.default_rng(seed + 7919 * step) return list(rng.integers(0, 2 ** 31, n)) return @app.cell(hide_code=True) def lib_avatar(sp): # ===== omni/avatar.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/avatar.py using only the imports this module actually needs. FPS = 25 MEL_HOP_SEC = sp.HOP / sp.SR # 11.6 ms per mel frame # mouth shapes: (open, wide, round, press) all roughly 0..1 VISEME_IDS = { "A": dict(open=0.95, wide=0.15, round=0.05, press=0.0), # AA AE AH AY "E": dict(open=0.55, wide=0.55, round=0.0, press=0.0), # EH EY IH IY "I": dict(open=0.35, wide=0.7, round=0.0, press=0.0), "O": dict(open=0.8, wide=-0.15, round=0.75, press=0.0), # AO OW OY AW "U": dict(open=0.45, wide=-0.25, round=0.9, press=0.0), # UW UH "ER": dict(open=0.5, wide=0.1, round=0.45, press=0.0), # ER "M": dict(open=0.02, wide=0.05, round=0.0, press=0.85), # M B P "F": dict(open=0.18, wide=0.35, round=0.0, press=0.35), # F V "TH": dict(open=0.25, wide=0.3, round=0.0, press=0.2), # TH DH "L": dict(open=0.45, wide=0.25, round=0.05, press=0.0), # L "R": dict(open=0.4, wide=0.0, round=0.5, press=0.0), # R "W": dict(open=0.25, wide=-0.2, round=0.85, press=0.0), # W "S": dict(open=0.22, wide=0.6, round=0.0, press=0.25), # S Z SH ZH CH JH "T": dict(open=0.3, wide=0.4, round=0.0, press=0.15), # T D N K G NG HH Y "SIL": dict(open=0.08, wide=0.0, round=0.0, press=0.5), } PHONE_TO_VISEME = { "AA": "A", "AE": "A", "AH": "A", "AY": "A", "AW": "O", "AO": "O", "OW": "O", "OY": "O", "EH": "E", "EY": "E", "IH": "E", "IY": "E", "UH": "U", "UW": "U", "ER": "ER", "M": "M", "B": "M", "P": "M", "F": "F", "V": "F", "TH": "TH", "DH": "TH", "L": "L", "R": "R", "W": "W", "S": "S", "Z": "S", "SH": "S", "ZH": "S", "CH": "S", "JH": "S", "T": "T", "D": "T", "N": "T", "K": "T", "G": "T", "NG": "T", "HH": "T", "Y": "T", } #: Palette. Warm, soft and low-contrast on purpose: hard flat fills are what made #: the earlier face read as a diagram. This is the surface an agent, a phone or a #: robot shows to a person, so it aims to be *appealing* -- soft shading, real #: eyes with catchlights, styled hair with a sheen -- and deliberately tasteful. SKIN = (247, 214, 190) SKIN_SHADE = (226, 180, 154) SKIN_DEEP = (198, 146, 122) BLUSH = (238, 152, 142) HAIR = (48, 30, 40) HAIR_HI = (188, 138, 132) BROW = (62, 40, 46) LIP = (205, 96, 108) LIP_DARK = (156, 68, 80) LIP_HI = (236, 162, 154) MOUTH_DARK = (74, 30, 38) TEETH = (250, 248, 245) TONGUE = (208, 110, 118) SCLERA = (252, 250, 248) IRIS = (130, 98, 72) IRIS_DARK = (72, 50, 38) LASH = (34, 20, 26) BG_TOP = (58, 42, 68) BG_BOTTOM = (104, 74, 98) def viseme_params(phones, durations, silence_lead=12, silence_tail=18): """Phoneme timeline (durations in mel frames) -> per-mel-frame parameters.""" keys = ["open", "wide", "round", "press"] frames = [] for vid, w in (("SIL", silence_lead),): frames += [VISEME_IDS[vid]] * max(0, w) for p, d in zip(phones, durations): vid = PHONE_TO_VISEME.get(p, "T") frames += [VISEME_IDS[vid]] * max(1, int(d)) frames += [VISEME_IDS["SIL"]] * max(0, silence_tail) arr = np.array([[f[k] for k in keys] for f in frames], np.float32) # smooth (co-articulation): a 3-tap and a 5-tap pass over the parameter track k5 = np.array([1, 2, 3, 2, 1], np.float32) k5 /= k5.sum() for j in range(arr.shape[1]): arr[:, j] = np.convolve(arr[:, j], k5, mode="same") return arr, keys def _rgb(c): return np.array(c, np.float32) def _vgrad(size, top, bottom, power=0.85): """A vertical gradient, as (size, size, 3) float.""" f = (np.linspace(0.0, 1.0, size) ** power)[:, None] return np.repeat((_rgb(top)[None, :] * (1 - f) + _rgb(bottom)[None, :] * f)[:, None, :], size, axis=1) def _smoothstep(x): x = np.clip(x, 0.0, 1.0) return x * x * (3.0 - 2.0 * x) def _radial(size, cx, cy, rx, ry, soft=0.6): """Smooth radial falloff, 1.0 at the centre -> 0.0 outside.""" yy, xx = np.mgrid[0:size, 0:size].astype(np.float32) d = np.sqrt(((xx - cx) / max(rx, 1e-3)) ** 2 + ((yy - cy) / max(ry, 1e-3)) ** 2) return _smoothstep((1.0 - d) / max(soft, 1e-3)) def _face_alpha(size, cx, cy, fw, fh, chin=0.55, feather=0.022): """A soft-edged face: an oval that tapers to a chin, not a circle.""" yy, xx = np.mgrid[0:size, 0:size].astype(np.float32) u = (xx - cx) / fw v = (yy - cy) / fh taper = 1.0 - chin * np.clip(v, 0.0, 1.0) ** 1.5 # jaw narrows below centre d = np.sqrt((u / np.maximum(taper, 1e-3)) ** 2 + v * v) return _smoothstep((1.0 - d) / feather)[..., None] def _almond(ex, ey, ew, eh, n=48, lift=0.30): """Points of an eye opening: a lens, with the upper lid higher than the lower.""" xs = np.linspace(-1.0, 1.0, n) up = -eh * (1.0 - xs * xs) ** 0.55 * (1.0 + lift * xs) low = eh * (1.0 - xs * xs) ** 0.70 top = [(ex + x * ew, ey + y) for x, y in zip(xs, up)] bot = [(ex + x * ew, ey + y) for x, y in zip(xs[::-1], low[::-1])] return top + bot def _tint(base, colour, amount): """Mix `base` (uint8 RGB array) toward `colour` by `amount` (float or array).""" a = np.clip(amount, 0.0, 1.0) if a.ndim == 2: a = a[..., None] return base * (1.0 - a) + _rgb(colour)[None, None, :] * a def _rounded_ellipse(draw, box, fill, outline=None, width=1): draw.ellipse(box, fill=fill, outline=outline, width=width) def _paste(img, layer): """Composite a PIL layer onto the float numpy image (RGB or RGBA).""" base = Image.fromarray(np.clip(img, 0, 255).astype(np.uint8)).convert("RGBA") base.alpha_composite(layer if layer.mode == "RGBA" else layer.convert("RGBA")) return np.asarray(base.convert("RGB"), np.float32) def _paste_masked(img, colour_img, mask): """Composite `colour_img` only where `mask` is set, with a soft edge. `Image.composite(a, b, mask)` returns RGB with every pixel filled -- the mask only chooses which *input* each pixel takes -- so pasting its result would paint the whole frame. An alpha channel is what scopes it to the mask. """ layer = colour_img.convert("RGBA") layer.putalpha(mask) return _paste(img, layer) # ---------------------------------------------------------------- sprite face -- #: A photoreal base portrait plus one image per viseme in which only the mouth #: differs. Generated by `sdxl_mouths.py`: img2img from a single base at a fixed seed, #: composited through a feathered mask, so the identity is *pixel-identical* outside #: the mouth and cannot drift between shapes. The four mouth parameters select and #: blend between these images exactly as they used to drive the drawn mouth, so the #: timing is still the model's own phoneme timeline. #: #: The sprites are published with the card (`assets/face/`) and fetched on demand, so a #: fresh sandbox gets them back; if they cannot be found, `render_frame` falls back to #: the drawn face rather than failing. SPRITE_DIR = Path("/marimo/sdxl/mouths") SPRITE_REPO = "hkblockchaintechnology/VLM-all-inclusive" SPRITE_PATH = "assets/face" #: Portrait crop of the 1024 px sprite: the head fills the frame. SPRITE_CROP = (0.200, 0.070, 0.800, 0.990) _SPRITES = {} def _sprite_file(name): p = SPRITE_DIR / f"mouth_{name}.png" if p.exists(): return p try: from huggingface_hub import hf_hub_download return Path(hf_hub_download(SPRITE_REPO, f"{SPRITE_PATH}/mouth_{name}.png", token=os.environ.get("HF_TOKEN") or None)) except Exception: return None def load_sprites(): """{viseme: PIL image} for whichever sprites can be found. Cached.""" if _SPRITES: return _SPRITES for n in VISEME_IDS: f = _sprite_file(n) if f is None: continue try: _SPRITES[n] = Image.open(f).convert("RGB") except Exception: pass return _SPRITES def _sprite_mix(params, names): """The three nearest visemes in mouth-parameter space, as normalised weights. Each axis is scaled by its range across the visemes first: otherwise `open`, which spans 0-1, would dominate `round`, which spans less. """ keys = ("open", "wide", "round", "press") anchors = np.array([[VISEME_IDS[n][k] for k in keys] for n in names], np.float32) span = np.maximum(np.ptp(anchors, axis=0), 1e-3) p = np.array([float(v) for v in params[:4]], np.float32) d = (np.abs(anchors - p) / span).sum(axis=1) order = np.argsort(d)[:3] w = np.exp(-(d[order] - d[order].min()) / 0.40) w = w / w.sum() return [(names[i], float(wi)) for i, wi in zip(order, w)] def _sprite_frame(params, size, t, energy, blink, ss): sprites = load_sprites() names = [n for n in VISEME_IDS if n in sprites] if not names: return None if len(names) > 1: acc = None for n, w in _sprite_mix(params, names): a = np.asarray(sprites[n], np.float32) * w acc = a if acc is None else acc + a frame = Image.fromarray(np.clip(acc, 0, 255).astype(np.uint8)) else: frame = sprites[names[0]] W, H = frame.size x0, y0, x1, y1 = SPRITE_CROP frame = frame.crop((int(x0 * W), int(y0 * H), int(x1 * W), int(y1 * H))) # a little life: the head drifts and breathes rather than sitting perfectly still bob = 0.012 * math.sin(t * 2.1) lean = 0.010 * math.sin(t * 1.3) zoom = 1.0 + 0.010 * math.sin(t * 0.9) + 0.010 * energy w, h = frame.size frame = frame.resize((int(w * zoom), int(h * zoom)), Image.LANCZOS) w2, h2 = frame.size canvas = Image.new("RGB", (w2, h2), (44, 33, 52)) canvas.paste(frame, (int((w2 - w) / 2 + lean * w2 * 0.5), int((h2 - h) / 2 - bob * h2 * 0.5))) return np.asarray(canvas.resize((size, size), Image.LANCZOS), np.uint8) def render_frame(params, size=384, t=0.0, energy=0.4, blink=0.0, ss=2): """One frame from the four mouth parameters. Sprite face if available, else drawn. `params` is (open, wide, round, press) -- the model's own phoneme timeline produces those, which is why the lips cannot disagree with the audio they came from. """ try: f = _sprite_frame(params, size, t, energy, blink, ss) except Exception: f = None if f is not None: return f return _procedural_frame(params, size=size, t=t, energy=energy, blink=blink, ss=ss) def _procedural_frame(params, size=384, t=0.0, energy=0.4, blink=0.0, ss=2): """params = (open, wide, round, press) -> an (size, size, 3) uint8 frame. A stylised character, driven by the four mouth parameters alone -- so the model's own phoneme timeline still moves every frame. What changed is the rendering: soft-shaded skin instead of flat fills, eyes with an iris gradient and catchlights, a modelled mouth, and hair with a sheen. The previous version was legible but read as a diagram, and this is the surface a person looks at. """ op, wide, rnd, press = [float(x) for x in params] S = float(size * ss) bob = 0.010 * S * math.sin(t * 2.1) lean = 0.009 * S * math.sin(t * 1.3) cx, cy = 0.5 * S + lean, 0.52 * S + bob fw, fh = 0.205 * S, 0.258 * S N = int(S) # ---------------------------------------------------------------- backdrop img = _vgrad(N, BG_TOP, BG_BOTTOM) # A luminous backdrop behind the head. Without it the dark hair sits on a dark # ground and the whole silhouette disappears -- which is exactly what happened # to the first version of this face. img = _tint(img, (196, 132, 176), 0.60 * _radial(N, cx, cy * 0.96, S * 0.52, S * 0.48, 1.05)) halo = _radial(N, cx, cy, S * (0.34 + 0.05 * energy), S * (0.34 + 0.05 * energy), 0.80) img = _tint(img, (255, 206, 226), 0.55 * halo * (0.55 + 0.45 * energy)) img = img * (1.0 - 0.52 * _radial(N, S / 2, S / 2, S * 0.74, S * 0.74, 1.35))[..., None] img = _tint(img, (255, 226, 238), 0.22 * _radial(N, cx, cy - fh * 0.2, S * 0.30, S * 0.28, 0.95)) # a warm key on one side and a cool fill on the other: one flat hue was called # "monotonous", and two light sources is what gives a backdrop colour to move through img = _tint(img, (255, 196, 150), 0.30 * _radial(N, S * 0.16, S * 0.30, S * 0.42, S * 0.42, 1.0)) img = _tint(img, (120, 170, 255), 0.26 * _radial(N, S * 0.88, S * 0.34, S * 0.42, S * 0.42, 1.0)) # ------------------------------------------------------- shoulders + neck sh = Image.new("RGBA", (N, N), (0, 0, 0, 0)) sd = ImageDraw.Draw(sh) sd.ellipse([cx - S * 0.44, cy + fh * 1.62, cx + S * 0.44, cy + fh * 3.4], fill=(48, 36, 58, 255)) sd.ellipse([cx - S * 0.30, cy + fh * 1.70, cx + S * 0.30, cy + fh * 3.2], fill=(64, 48, 74, 255)) img = _paste(img, sh) nk = Image.new("RGBA", (N, N), (0, 0, 0, 0)) ImageDraw.Draw(nk).rounded_rectangle( [cx - S * 0.072, cy + fh * 0.66, cx + S * 0.072, cy + fh * 1.98], radius=int(S * 0.055), fill=SKIN_SHADE + (255,)) img = _paste(img, nk) img = _tint(img, SKIN_DEEP, 0.26 * _radial(N, cx, cy + fh * 1.4, S * 0.09, S * 0.30, 0.9)) # --------------------------------------------------------- hair, behind hair = Image.new("RGBA", (N, N), (0, 0, 0, 0)) hd = ImageDraw.Draw(hair) # rim light: the same silhouette, a little larger, in a bright tone -- the hair # then reads as hair rather than as a hole in the background # the mass behind the head stops at the jaw; the length comes from the locks below hd.ellipse([cx - fw * 1.36, cy - fh * 1.36, cx + fw * 1.36, cy + fh * 1.06], fill=HAIR_HI + (255,)) hd.ellipse([cx - fw * 1.30, cy - fh * 1.32, cx + fw * 1.30, cy + fh * 1.02], fill=HAIR + (255,)) # side locks rather than a solid mass: hair should frame the face, not stand in # for the neck. A full rectangle here left a dark void under the chin. for sx in (-1, 1): hd.ellipse([cx + sx * fw * 0.74 - fw * 0.46, cy - fh * 0.60, cx + sx * fw * 0.74 + fw * 0.46, cy + fh * 2.02], fill=HAIR + (255,)) # and a sheen band across the crown hd.ellipse([cx - fw * 0.92, cy - fh * 1.22, cx + fw * 0.30, cy - fh * 0.30], fill=HAIR_HI + (120,)) hair = hair.filter(ImageFilter.GaussianBlur(max(1.5, S * 0.011))) # strands: a few dozen thin arcs in a lighter tone, so the hair reads as hair strands = Image.new("RGBA", (N, N), (0, 0, 0, 0)) sdw = ImageDraw.Draw(strands) hrng = np.random.default_rng(4242) for _ in range(46): a = hrng.uniform(-1.0, 1.0) x0 = cx + a * fw * 1.15 y0 = cy - fh * hrng.uniform(0.55, 1.25) x1 = x0 + fw * hrng.uniform(-0.22, 0.22) y1 = y0 + fh * hrng.uniform(0.9, 2.1) sdw.line([(x0, y0), (x0 + (x1 - x0) * 0.5, (y0 + y1) / 2), (x1, y1)], fill=HAIR_HI + (int(hrng.uniform(40, 105)),), width=max(1, int(S * hrng.uniform(0.0018, 0.0042))), joint="curve") strands = strands.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.0022))) hair_alpha = hair.split()[-1].filter(ImageFilter.GaussianBlur(max(1.0, S * 0.006))) strands.putalpha(Image.fromarray(np.minimum(np.asarray(strands.split()[-1]), np.asarray(hair_alpha)).astype(np.uint8))) img = _paste(img, hair) img = _paste(img, strands) # a parting: "the hair covers the forehead, it should be styled differently". A # scalp-toned wedge at the centre-top splits the mass and opens the face. part = Image.new("RGBA", (N, N), (0, 0, 0, 0)) pdw = ImageDraw.Draw(part) for k in range(3): w = fw * (0.085 + 0.030 * k) pdw.polygon([(cx - w, cy - fh * 1.02), (cx + w * 0.30, cy - fh * 1.44), (cx + w * 0.62, cy - fh * 1.00)], fill=(84, 52, 58, 130)) part = part.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.006))) img = _paste(img, part) # ------------------------------------------------------------------- face alpha = _face_alpha(N, cx, cy, fw, fh) skin = _vgrad(N, SKIN, SKIN_SHADE, 1.35) skin = _tint(skin, (255, 248, 238), 0.66 * _radial(N, cx - fw * 0.34, cy - fh * 0.44, fw * 0.92, fh * 0.82, 0.92)) skin = _tint(skin, BLUSH, 0.34 * _radial(N, cx - fw * 0.60, cy + fh * 0.24, fw * 0.42, fh * 0.26, 0.95)) skin = _tint(skin, BLUSH, 0.34 * _radial(N, cx + fw * 0.60, cy + fh * 0.24, fw * 0.42, fh * 0.26, 0.95)) skin = _tint(skin, SKIN_DEEP, 0.30 * _radial(N, cx + fw * 0.74, cy + fh * 0.30, fw * 0.64, fh * 0.74, 1.1)) # --- form shading. "Flat, lacking shading and depth" was the one criticism the # vision model made twice, of two different versions. Depth comes from four # shapes: the perimeter rolling away, the brow ridge casting down, the sides of # the nose, and the jaw. Each is a soft tint rather than a drawn line, because a # drawn line reads as a cartoon. skin = _tint(skin, SKIN_DEEP, 0.58 * (1.0 - _radial(N, cx, cy, fw * 1.04, fh * 1.06, 0.66))) for sx in (-1, 1): bx = cx + sx * fw * 0.257 skin = _tint(skin, (176, 122, 108), 0.44 * _radial(N, bx, cy - fh * 0.040, fw * 0.20, fh * 0.080, 1.0)) skin = _tint(skin, (198, 142, 126), 0.26 * _radial(N, cx + sx * fw * 0.075, cy + fh * 0.055, fw * 0.055, fh * 0.16, 1.0)) skin = _tint(skin, (214, 158, 140), 0.30 * _radial(N, cx, cy + fh * 0.70, fw * 0.62, fh * 0.20, 1.0)) rng = np.random.default_rng(int(cx * 31 + cy * 17) & 0xFFFF) skin = skin + rng.normal(0.0, 2.1, (N, N, 1)).astype(np.float32) * alpha img = img * (1.0 - alpha) + skin * alpha # ------------------------------------------------------------------ brows br = Image.new("RGBA", (N, N), (0, 0, 0, 0)) bd = ImageDraw.Draw(br) for sx in (-1, 1): # brows sit a little wider than the pupils and follow the eye line, which # moved up: they must be defined here rather than after the eyes, because # that is where `ex_off` is introduced. bx = cx + sx * fw * 0.257 by = cy - fh * (0.31 + 0.06 * op) for k in range(3): w = fw * (0.195 - 0.015 * k) bd.arc([bx - w, by - fh * (0.10 + 0.012 * k), bx + w, by + fh * 0.16], start=200 + 25 * sx, end=340 + 25 * sx, fill=BROW + (230,), width=max(3, int(S * (0.017 - 0.0032 * k)))) img = _paste(img, br.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.0035)))) # ------------------------------------------------------------------- eyes # Proportions, not taste: an eye is roughly one fifth of the face width and the # eye line sits near the middle of the head. The first version had # `ew = 0.255*fw` -- an eye half as wide as the face -- which is why a vision # model described the features as "exaggerated and lacking natural proportions". eh = fh * 0.105 * (1.0 - 0.92 * blink) ew = fw * 0.145 ey = cy - fh * 0.12 ex_off = fw * 0.255 cl = Image.new("RGBA", (N, N), (0, 0, 0, 0)) cd = ImageDraw.Draw(cl) if eh < fh * 0.014: # a blink: just the lash line for sx in (-1, 1): ex = cx + sx * ex_off cd.line([(ex - ew, ey), (ex, ey - fh * 0.012), (ex + ew, ey)], fill=LASH + (255,), width=max(2, int(S * 0.009)), joint="curve") img = _paste(img, cl) else: shade = Image.new("RGBA", (N, N), (0, 0, 0, 0)) sdr = ImageDraw.Draw(shade) for sx in (-1, 1): ex = cx + sx * ex_off sdr.ellipse([ex - ew * 1.30, ey - eh * 2.4, ex + ew * 1.30, ey + eh * 1.5], fill=(150, 96, 104, 95)) img = _paste(img, shade.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.012)))) scl = Image.new("RGB", (N, N), SCLERA) for sx in (-1, 1): ex = cx + sx * ex_off mask = Image.new("L", (N, N), 0) ImageDraw.Draw(mask).polygon(_almond(ex, ey, ew, eh), fill=255) mask = mask.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.0022))) iris = Image.new("RGB", (N, N), IRIS) idr = ImageDraw.Draw(iris) ir = eh * 0.98 icx, icy = ex, ey + eh * 0.10 for k in range(10): f = 1.0 - k / 10.0 col = tuple(int(IRIS[c] * f + IRIS_DARK[c] * (1.0 - f)) for c in range(3)) idr.ellipse([icx - ir * f, icy - ir * f, icx + ir * f, icy + ir * f], fill=col) idr.ellipse([icx - ir * 0.42, icy - ir * 0.42, icx + ir * 0.42, icy + ir * 0.42], fill=(28, 20, 22)) idr.ellipse([icx - ir * 0.62, icy - ir * 0.68, icx - ir * 0.14, icy - ir * 0.18], fill=(255, 252, 250)) idr.ellipse([icx + ir * 0.16, icy + ir * 0.22, icx + ir * 0.46, icy + ir * 0.50], fill=(236, 232, 228)) img = _paste_masked(img, scl, mask) # the sclera fills the almond img = _paste_masked(img, iris, mask) # the iris sits on top of it for sx in (-1, 1): ex = cx + sx * ex_off xs = np.linspace(-1, 1, 24) cd.line([(ex + x * ew, ey - eh * (1 - x * x) ** 0.55 * (1 + 0.30 * x)) for x in xs], fill=LASH + (255,), width=max(2, int(S * 0.008)), joint="curve") cd.line([(ex - ew * 1.02, ey + eh * 0.06), (ex, ey - eh * 0.98), (ex + ew * 1.06, ey - eh * 0.22), (ex + ew * 1.42, ey - eh * 0.72)], fill=LASH + (255,), width=max(2, int(S * 0.006)), joint="curve") cd.line([(ex - ew, ey + eh * 0.10), (ex, ey + eh * 0.95), (ex + ew, ey + eh * 0.16)], fill=(120, 78, 82, 150), width=max(1, int(S * 0.0035)), joint="curve") for k in range(3): t0 = 0.72 + 0.10 * k oy = ey - eh * (1 - t0 * t0) ** 0.55 * (1 + 0.30 * t0) + eh * 0.05 cd.line([(ex + sx * ew * t0, oy), (ex + sx * ew * (t0 + 0.16), oy - eh * (0.34 + 0.14 * k))], fill=LASH + (255,), width=max(1, int(S * 0.0045))) cd.arc([ex - ew * 1.05, ey - eh * 2.5, ex + ew * 1.05, ey + eh * 0.55], start=205, end=335, fill=(146, 96, 96, 110), width=max(1, int(S * 0.004))) img = _paste(img, cl) # ------------------------------------------------------------------- nose ns = Image.new("RGBA", (N, N), (0, 0, 0, 0)) ndr = ImageDraw.Draw(ns) ndr.line([(cx + fw * 0.02, cy - fh * 0.04), (cx - fw * 0.03, cy + fh * 0.20), (cx + fw * 0.05, cy + fh * 0.21)], fill=SKIN_DEEP + (150,), width=max(2, int(S * 0.007)), joint="curve") ndr.ellipse([cx - fw * 0.05, cy + fh * 0.20, cx + fw * 0.07, cy + fh * 0.245], fill=(196, 140, 122, 120)) img = _paste(img, ns.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.006)))) img = _tint(img, (255, 248, 240), 0.30 * _radial(N, cx - fw * 0.02, cy + fh * 0.06, fw * 0.075, fh * 0.20, 0.9)) # ------------------------------------------------------------------ mouth my = cy + S * 0.145 mw = max(S * 0.030, S * 0.061 * (1.0 + 0.34 * wide - 0.20 * rnd)) gap = S * (0.0016 + 0.070 * op) * (1.0 - 0.55 * press) up_t = S * 0.0132 * (1.0 + 0.30 * (1.0 - press)) * (1.0 + 0.18 * rnd) lo_t = S * 0.0190 * (1.0 - 0.45 * press) * (1.0 + 0.20 * rnd) lip = Image.new("RGBA", (N, N), (0, 0, 0, 0)) ld = ImageDraw.Draw(lip) xs = np.linspace(-1, 1, 41) up_y = -up_t * (1 - xs * xs) ** 0.55 + up_t * 0.42 * np.exp(-(xs / 0.24) ** 2) lo_y = lo_t * (1 - xs * xs) ** 0.62 upper = [(cx + x * mw, my - gap * 0.5 + y) for x, y in zip(xs, up_y)] lower = [(cx + x * mw, my + gap * 0.5 + y) for x, y in zip(xs[::-1], lo_y[::-1])] if rnd > 0.05: ld.ellipse([cx - mw * 1.22, my - (up_t + lo_t) * 1.5, cx + mw * 1.22, my + (up_t + lo_t) * 1.5], fill=(188, 120, 112, int(90 * rnd))) if gap > S * 0.004: ld.ellipse([cx - mw * 0.86, my - gap * 0.62, cx + mw * 0.86, my + gap * 0.62], fill=MOUTH_DARK + (255,)) ld.polygon(upper, fill=LIP + (255,)) ld.polygon(lower, fill=LIP + (255,)) ld.polygon([(cx + x * mw * 0.86, my + gap * 0.5 + y * 0.92) for x, y in zip(xs, lo_y)], fill=LIP_HI + (255,)) ld.line([(cx - mw * 0.96, my), (cx, my + S * 0.0015), (cx + mw * 0.96, my)], fill=LIP_DARK + (255,), width=max(1, int(S * 0.005)), joint="curve") if gap > S * 0.004: # Teeth and tongue were filling the whole opening, leaving no dark interior: a # wide-open mouth showed almost no shadow, which reads as a blocked throat and # also broke the test that says an open mouth must expose more dark interior. th = gap * 0.23 ld.rounded_rectangle([cx - mw * 0.80, my - gap * 0.60, cx + mw * 0.80, my - gap * 0.60 + th], radius=int(S * 0.006), fill=TEETH + (255,)) if op > 0.35: ld.rounded_rectangle([cx - mw * 0.72, my + gap * 0.28, cx + mw * 0.72, my + gap * 0.62], radius=int(S * 0.006), fill=TEETH + (245,)) tr = mw * 0.32 * min(1.0, op) ld.ellipse([cx - tr, my + gap * 0.34, cx + tr, my + gap * 1.05], fill=TONGUE + (255,)) gloss = Image.new("L", (N, N), 0) ImageDraw.Draw(gloss).ellipse([cx - mw * 0.34, my + gap * 0.5 + lo_t * 0.10, cx - mw * 0.02, my + gap * 0.5 + lo_t * 0.62], fill=190) gloss = gloss.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.006))) lip = Image.composite(Image.new("RGBA", lip.size, (255, 246, 242, 255)), lip, gloss) img = _paste(img, lip.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.0018)))) crease = Image.new("RGBA", (N, N), (0, 0, 0, 0)) ImageDraw.Draw(crease).arc([cx - mw * 1.10, my - (up_t + lo_t) * 1.25, cx + mw * 1.10, my + (up_t + lo_t) * 2.1], start=25, end=155, fill=(180, 116, 110, 120), width=max(1, int(S * 0.0035))) img = _paste(img, crease.filter(ImageFilter.GaussianBlur(max(1.0, S * 0.004)))) img = _tint(img, (196, 138, 122), 0.34 * _radial(N, cx, my + lo_t * 1.5, mw * 0.86, lo_t * 1.1, 1.0)) img = _tint(img, (188, 130, 116), 0.26 * _radial(N, cx, cy + fh * 0.95, fw * 0.72, fh * 0.16, 1.0)) out = np.clip(img, 0, 255).astype(np.uint8) if ss > 1: return np.asarray(Image.fromarray(out).resize((size, size), Image.LANCZOS), np.uint8) return out def mel_frames(mel_log, n_mels: int | None = None): """Normalise a log-mel to (frames, n_mels). Rule, in order: an explicit ``n_mels`` wins; otherwise the axis matching the default mel count is the mel axis; otherwise the shorter axis is the mel axis, because an utterance always has more frames than bins. """ m = np.asarray(mel_log, np.float32) if m.ndim != 2: raise ValueError(f"mel must be 2-D, got {m.shape}") target = int(n_mels or sp.N_MELS) if m.shape[1] == target: return m if m.shape[0] == target: return m.T return m if m.shape[1] < m.shape[0] else m.T def energy_track(mel_log, n_frames, n_mels: int | None = None): """Cheap loudness envelope from log-mel, resampled to the frame count.""" m = mel_frames(mel_log, n_mels) e = np.exp(m).sum(0) if m.ndim == 2 else np.ones(n_frames) e = e / (np.percentile(e, 95) + 1e-6) e = np.clip(e, 0, 1.6) xp = np.linspace(0, 1, len(e)) return np.interp(np.linspace(0, 1, n_frames), xp, e) def pad_wav_to_timeline(wav, lead_frames=12, tail_frames=18): """Match the audio to the viseme timeline. `viseme_params` adds a short silence lead/tail so the mouth closes before and after speech. The wav must cover the same span, otherwise ffmpeg (with -shortest) stops reading video frames and the pipe breaks. """ wav = np.asarray(wav, np.float32) lead = int(lead_frames * sp.HOP) tail = int(tail_frames * sp.HOP) return np.concatenate([np.zeros(lead, np.float32), wav, np.zeros(tail, np.float32)]) def render_talking_head(mel_log, phones, durations, out_path, wav_path=None, fps=FPS, size=384, seed=0, log=print, max_sec=20.0, n_mels: int | None = None): """Rasterise the talking head for a phoneme timeline and mux to mp4.""" params, keys = viseme_params(phones, durations) if mel_log is not None: mel_log = mel_frames(mel_log, n_mels) # (frames, n_mels) n_mel = params.shape[0] dur_sec = n_mel * MEL_HOP_SEC n_frames = max(1, int(dur_sec * fps)) n_frames = min(n_frames, int(max_sec * fps)) rng = np.random.default_rng(seed) # blink schedule: a blink every ~2.6-4.6 s, closing over ~3 frames blink_at = [] t_ = 1.5 + float(rng.random()) while t_ < dur_sec: blink_at.append(t_) t_ += 2.2 + 1.6 * float(rng.random()) energy = (energy_track(mel_log, n_mel, n_mels) if mel_log is not None else np.full(n_mel, 0.5)) out_path = Path(out_path) have_ffmpeg = shutil.which("ffmpeg") is not None frames = [] proc = None if have_ffmpeg: cmd = ["ffmpeg", "-y", "-loglevel", "error", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", f"{size}x{size}", "-r", str(fps), "-i", "-"] if wav_path: cmd += ["-i", str(wav_path), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "160k", "-shortest"] else: cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-preset", "veryfast", "-crf", "20"] cmd += [str(out_path)] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) for i in range(n_frames): t = i / fps f_idx = min(n_mel - 1, int(t / MEL_HOP_SEC)) p = params[f_idx] bl = 0.0 for bt in blink_at: dt = t - bt if 0 <= dt < 0.16: bl = max(bl, 1.0 - abs(dt - 0.08) / 0.08) frame = render_frame(p, size=size, t=t, energy=float(energy[f_idx]), blink=bl) frames.append(frame) if proc: try: proc.stdin.write(frame.tobytes()) except BrokenPipeError: # ffmpeg stopped early (audio shorter) log("ffmpeg closed the pipe early; the movie will be short") proc = None if proc: proc.stdin.close() err = proc.stderr.read().decode(errors="replace") rc = proc.wait() if rc != 0: log(f"ffmpeg failed: {err[:400]}") else: log(f"movie written: {out_path} ({n_frames} frames, {dur_sec:.1f}s)") return out_path, np.stack(frames) if frames else None def movie_from_mel(mel_log, phones, durations, out_dir, name, fps=FPS, size=384, log=print): """Full path: (mel, phonemes) -> wav + talking-head mp4.""" out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) wav_path = out_dir / f"{name}.wav" mp4_path = out_dir / f"{name}.mp4" wav = pad_wav_to_timeline(griffin_or_vocoder(mel_log)) sp.write_wav(wav_path, wav) return render_talking_head(mel_log, phones, durations, mp4_path, wav_path=wav_path, fps=fps, size=size, log=log), wav_path, mp4_path def griffin_or_vocoder(mel_log, vocoder=None, device="cuda", n_mels: int | None = None): """log-mel (either orientation) -> waveform, using the learned vocoder.""" import torch m = mel_frames(mel_log, n_mels) # (T, n_mels) if vocoder is None: return sp.griffin_lim(m.T, iters=48, n_mels=m.shape[1]) with torch.no_grad(): x = torch.as_tensor(m, dtype=torch.float32, device=device).transpose(0, 1)[None] # (1, M, T) return vocoder(x)[0].float().cpu().numpy() return @app.cell(hide_code=True) def lib_train_vlm(OmniVLM, VLMConfig, count_params, od, scenes): # ===== omni/train_vlm.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/train_vlm.py using only the imports this module actually needs. def preset(name: str) -> VLMConfig: """Sizes that keep the PRO 6000 busy without wasting it.""" table = { "nano": dict(d_model=256, n_layers=4, n_heads=4, vision_d_model=192, vision_layers=3, vision_heads=3), "small": dict(d_model=384, n_layers=6, n_heads=6, vision_d_model=256, vision_layers=4, vision_heads=4), "base": dict(d_model=512, n_layers=8, n_heads=8, vision_d_model=384, vision_layers=6, vision_heads=6), "large": dict(d_model=768, n_layers=12, n_heads=12, vision_d_model=512, vision_layers=8, vision_heads=8), "xl": dict(d_model=1024, n_layers=16, n_heads=16, vision_d_model=640, vision_layers=10, vision_heads=10), "xxl": dict(d_model=1280, n_layers=24, n_heads=16, vision_d_model=768, vision_layers=12, vision_heads=12), } return VLMConfig(**table[name]) def train(cfg: dict, store, vocab, model: OmniVLM | None = None, start_step: int = 0, resume: dict | None = None, log=print, device="cuda"): d = cfg torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True torch.backends.cudnn.benchmark = True ds = scenes.SceneDataset(seed=d["data_seed"], px=d["img_size"]) if model is None: model = OmniVLM(cfg=VLMConfig.from_dict(d["model"]), n_words=vocab.n_words, n_phones=vocab.n_phones) model = model.to(device) if d.get("channels_last", True): model = model.to(memory_format=torch.channels_last) n_par = count_params(model) opt = torch.optim.AdamW(model.parameters(), lr=d["lr"], betas=(0.9, 0.95), weight_decay=d["wd"], fused=True) if resume and resume.get("optim"): load_opt_flat(opt, resume["optim"]) log(f"restored optimizer state ({len(resume['optim'])} tensors)") sched = lambda s: min(1.0, (s + 1) / d["warmup"]) * (0.5 * (1 + np.cos(np.pi * min(1.0, s / d["steps"])))) scaler = torch.amp.GradScaler("cuda", enabled=d.get("amp_dtype") == "fp16") amp = torch.autocast("cuda", dtype=torch.bfloat16 if d.get("amp_dtype", "bf16") == "bf16" else torch.float16) log(f"stage=vlm preset={d['preset']} params={n_par/1e6:.1f}M batch={d['batch']} " f"accum={d['accum']} steps={d['steps']} start={start_step}") # ---- background prefetcher: renders the *next* batches while the GPU works def index_iter(): st = start_step while st < d["steps"] + 2: for micro in range(d["accum"]): yield od.batch_indices_for_step(st * d["accum"] + micro, d["batch"], d["data_seed"]) st += 1 prefetch = od.Prefetcher( lambda idx: scenes.spec_batch(idx, seed=d["data_seed"], px=d["img_size"]), index_iter(), depth=6) log(f"stage=vlm preset={d['preset']} params={n_par/1e6:.1f}M batch={d['batch']} " f"accum={d['accum']} steps={d['steps']} start={start_step} device={torch.cuda.get_device_name(0)}") t0 = time.time() step = start_step losses = [] timers = {"fetch": 0.0, "collate": 0.0, "fwd_bwd": 0.0, "opt": 0.0} while step < d["steps"]: opt.zero_grad(set_to_none=True) accs = [] for micro in range(d["accum"]): t_f = time.time() spec = next(prefetch) t_c = time.time() batch = od.collate_specs(vocab, spec, device=device, px=d["img_size"], ss=d.get("ss", 1)) t_g = time.time() with amp: out = model.forward_text(batch["ids"], images=batch["images"], targets=batch["targets"], loss_mask=batch["mask"]) loss = out["loss"] / d["accum"] scaler.scale(loss).backward() losses.append(float(out["loss"].detach())) accs.append(float(out["acc"])) timers["fetch"] += t_c - t_f timers["collate"] += t_g - t_c timers["fwd_bwd"] += time.time() - t_g torch.cuda.synchronize() t_o = time.time() scaler.unscale_(opt) torch.nn.utils.clip_grad_norm_(model.parameters(), d["clip"]) lr = d["lr"] * sched(step) for g in opt.param_groups: g["lr"] = lr scaler.step(opt) scaler.update() torch.cuda.synchronize() timers["opt"] += time.time() - t_o step += 1 if step % d["log_every"] == 0 or step == start_step + 1: sps = (time.time() - t0) / max(1, step - start_step) n_st = d["log_every"] * d["accum"] store.log(stage="vlm", step=step, loss=float(np.mean(losses[-n_st:])), acc=float(np.mean(accs)), lr=lr, sec_per_step=round(sps, 3), samples_per_s=round(d["batch"] * d["accum"] / sps, 1), params=n_par, ms_fetch=round(1000 * timers["fetch"] / n_st, 2), ms_collate=round(1000 * timers["collate"] / n_st, 2), ms_fwdbwd=round(1000 * timers["fwd_bwd"] / n_st, 2), ms_opt=round(1000 * timers["opt"] / n_st, 2), gpu_gb=round(torch.cuda.max_memory_allocated() / 1e9, 2)) timers = {"fetch": 0.0, "collate": 0.0, "fwd_bwd": 0.0, "opt": 0.0} if store.should_checkpoint(step) or step >= d["steps"]: store.save(stage="vlm", step=step, payload={"model": model.state_dict(), "optim": _opt_flat(opt), "meta": {"preset": d["preset"], "model": d["model"], "n_words": vocab.n_words, "n_phones": vocab.n_phones}}, metrics={"loss": float(np.mean(losses[-d["log_every"] * d["accum"]:]))}) _showcase(model, vocab, ds, step, log, device, px=d["img_size"]) if d.get("minutes") and (time.time() - t0) > d["minutes"] * 60: log(f"time budget reached at step {step}") break prefetch.close() return model, step def _opt_flat(opt): out = {} for k, v in opt.state_dict()["state"].items(): for k2, v2 in v.items(): if torch.is_tensor(v2): out[f"{k}.{k2}"] = v2 return out def load_opt_flat(opt, flat): st = opt.state_dict() for k, v in flat.items(): i, k2 = k.split(".", 1) st["state"].setdefault(int(i), {})[k2] = v opt.load_state_dict(st) @torch.no_grad() def _showcase(model, vocab, ds, step, log, device, n=4, px=128): model.eval() spec = scenes.spec_batch(od.sample_metric_batches(step, n), seed=ds.seed, px=px) batch = od.collate_specs(vocab, spec, device=device, px=px) q_len = None # teacher-forced accuracy out = model.forward_text(batch["ids"], images=batch["images"], targets=batch["targets"], loss_mask=batch["mask"]) # greedy generation from the question only gen_in = batch["ids"][:, :batch["ids"].shape[1]] for i, (q, a) in enumerate(zip(batch["questions"], batch["answers"])): seq, st = od.encode_qa(vocab, q, a) prefix = torch.tensor(seq[:st], device=device)[None] g = model.generate_text(prefix, images=batch["images"][i:i + 1], max_new=24) pred = " ".join(vocab.ids_to_words(g[0])) log(f" Q: {q}\n want: {a}\n got : {pred}") log(f" teacher-forced acc @{step}: {float(out['acc']):.3f}") model.train() return @app.cell(hide_code=True) def lib_train_speech(VLMConfig, od, scenes, sp, textmod): # ===== omni/train_speech.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/train_speech.py using only the imports this module actually needs. def lj_and_g2p(root="/marimo/data/LJSpeech-1.1", dict_path="/marimo/data/cmudict.dict"): return sp.LJSpeech(root), textmod.G2P(dict_path) def clip_indices_for_step(step, batch, seed, n_clips, offset=0): rng = np.random.default_rng(seed * 7919 + step + offset) return list(rng.integers(0, n_clips, batch)) def load_batch(lj, idxs, max_sec=None, crop_sec=None, seed=0): """Deterministic crop so a resumed run sees the same audio.""" out = [] for i, cid in enumerate(idxs): wav, item = lj.load(int(cid)) if crop_sec: n = int(crop_sec * sp.SR) if len(wav) > n: rng = np.random.default_rng(seed * 104729 + int(cid)) s = int(rng.integers(0, len(wav) - n)) wav = wav[s:s + n] if max_sec: wav = wav[: int(max_sec * sp.SR)] out.append((wav, item["text"])) return out def pad_mels(mels, device): T = max(m.shape[-1] for m in mels) out = torch.full((len(mels), sp.N_MELS, T), float(np.log(1e-5))) for i, m in enumerate(mels): out[i, :, :m.shape[-1]] = torch.from_numpy(m) return out.to(device) # --------------------------------------------------------------------- ear def lr_warmup_cosine(store, steps: int, warmup: int = 200, floor: float = 0.05): """The schedule speech and the vocoder should always have had. Both stages shipped with a *constant* learning rate, while the language stages have had warmup + cosine from the beginning. A constant rate on a reconstruction loss is what let the vocoder spike at step 102k (0.86 -> 2.65) and then drift from 0.82 to 0.98 late in the run. The horizon comes from `store`, which records it in the run's own state, so raising the launcher's `--steps` cannot silently re-heat the rate and knock a converged model off its minimum. Returns (schedule, note); the note is worth logging when the launcher now asks for more steps than the run was shaped for. """ total = store.schedule_total(steps) warm = max(1, int(warmup)) floor = max(0.0, float(floor)) def sched(step: int) -> float: # The cosine is floored on purpose. Without it a run that passes its # horizon decays to *exactly* zero, and then keeps burning GPU computing # gradients that are multiplied by zero -- the loss simply stops moving, # which is indistinguishable from convergence. The vocoder sat frozen at # stft 0.9130 for 30 000 steps that way, at lr = 0.000e+00, while looking # perfectly healthy from the outside. The warmup still ramps from zero; # only the tail is floored. cosine = 0.5 * (1 + np.cos(np.pi * min(1.0, step / total))) return min(1.0, (step + 1) / warm) * max(floor, cosine) return sched, store.budget_note(steps) def stage_ear(cfg, store, log=print, device="cuda"): lj, g2p = lj_and_g2p() ev = sp.EarVocab() d = cfg ear = sp.Ear(n_out=len(ev)).to(device) opt = torch.optim.AdamW(ear.parameters(), lr=d.get("ear_lr") or d.get("lr", 3e-4), weight_decay=0.01, fused=True) ctc = torch.nn.CTCLoss(blank=0, zero_infinity=True) start = 0 if d["resume"] != "never": ck = store.load_latest(tag="model") if ck and ck.get("payload"): ear.load_state_dict(ck["payload"]) start = ck["step"] ok = store.load_latest(tag="optim") if ok and ok.get("payload"): from omni.train_vlm import load_opt_flat load_opt_flat(opt, ok["payload"]) log(f"RESUMED {store.run_id} at step {start}") n_par = sum(p.numel() for p in ear.parameters()) log(f"stage=ear params={n_par/1e6:.2f}M batch={d['batch']} steps={d['steps']} start={start}") t0 = time.time() step = start losses = [] while step < d["steps"]: idxs = clip_indices_for_step(step, d["batch"], d.get("sp_seed", 1), len(lj), offset=start) batch = load_batch(lj, idxs, max_sec=d.get("ear_max_sec", 6.0), seed=d.get("sp_seed", 1)) mels = [sp.mel_spectrogram(torch.from_numpy(w)[None])[0].numpy() for w, _ in batch] mel_t = pad_mels(mels, device) lengths = torch.tensor([m.shape[-1] for m in mels], device=device) targets = [ev.encode(g2p.text_to_phones(t))[:400] or [0] for _, t in batch] Lt = max(len(t) for t in targets) tgt = torch.zeros((len(targets), Lt), dtype=torch.long) for i, t in enumerate(targets): tgt[i, :len(t)] = torch.tensor(t) tgt = tgt.to(device) logp = ear(mel_t) in_len = torch.clamp((lengths // 4).long(), max=logp.shape[1]) loss = ctc(logp.transpose(0, 1), tgt, in_len, torch.tensor([len(t) for t in targets], device=device)) opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(ear.parameters(), 1.0) opt.step() losses.append(float(loss.detach())) step += 1 if step % d.get("log_every", 20) == 0: sps = (time.time() - t0) / max(1, step - start) store.log(stage="ear", step=step, ctc_loss=float(np.mean(losses[-d.get("log_every", 20):])), sec_per_step=round(sps, 3)) log(f"ear step {step:6d} ctc {np.mean(losses[-d.get('log_every',20):]):.3f}") if store.should_checkpoint(step) or step >= d["steps"]: store.save(stage="ear", step=step, payload={"model": ear.state_dict(), "optim": _flat(opt), "meta": {"kind": "ear", "n_out": len(ev)}}, metrics={"ctc": float(np.mean(losses[-d.get("log_every", 20):]))}) log(f" ear sample: " + _ear_sample(ear, lj, g2p, ev, device)) if d.get("minutes") and (time.time() - t0) > d["minutes"] * 60: break return ear, step def _flat(opt): out = {} for k, v in opt.state_dict()["state"].items(): for k2, v2 in v.items(): if torch.is_tensor(v2): out[f"{k}.{k2}"] = v2 return out @torch.no_grad() def _ear_sample(ear, lj, g2p, ev, device, n=1): out = [] for i in range(n): wav, item = lj.load(int(20 + i)) mel = sp.mel_spectrogram(torch.from_numpy(wav)[None].to(device)) pred = ear(mel)[0].argmax(-1).cpu().numpy() hyp = ev.decode(pred) if hasattr(ev, "decode") else [] phones = [] prev = None for p in pred: s = ev.itos[int(p)] if s != "" and s != prev: phones.append(s) prev = s out.append(f"want[{' '.join(g2p.text_to_phones(item['text'])[:18])}] " f"got[{' '.join(phones[:18])}]") return " | ".join(out) # ------------------------------------------------------------------- align def write_alignments_atomic(path, data): """Write the alignment cache atomically. A trainer starting up while this file is half-written sees a truncated gzip and refuses to start. `os.replace` means a reader gets either the old file or the new one, never a partial one -- the same rule the checkpoint writer follows, and worth having because alignment now legitimately happens while training runs. """ path = Path(path) tmp = path.with_suffix(path.suffix + ".part") with gzip.open(tmp, "wt") as f: json.dump(data, f) os.replace(tmp, path) return path def align_path(run_id, step): """Zero-padded, because `load_alignments` takes the last name in sorted order: in plain string order 99999 sorts *after* 128503, which would serve a stale alignment once the ear passes 100k steps.""" return Path(f"/marimo/runs/shared/align_{run_id}_{int(step):09d}.json.gz") def stage_align(cfg, store, log=print, device="cuda"): """Forced-align the corpus with the trained ear; cached locally and mirrored to the Hub so a fresh sandbox never recomputes it.""" lj, g2p = lj_and_g2p() ev = sp.EarVocab() ear = sp.Ear(n_out=len(ev)).to(device) # Align with the *newest ear*, wherever it lives. This used to look for a # checkpoint inside the align run's own directory, which never has one -- so a # good ear and a stale alignment cache coexisted, and the speech stage kept # learning durations from an ear with a few thousand steps. The alignment is # the entire supervision for how long each phone lasts, and therefore for lip # timing: it is only as good as the ear that produced it. import omni.hub as hub from omni.train import token_from_env ear_store, _ear_state = hub.open_stage_run(cfg["hf_repo"], "ear", token_from_env()) ck = ear_store.load_latest(tag="model") if ear_store else None if not ck: raise SystemExit("no ear checkpoint yet - run --stage ear first") ear.load_state_dict(ck["payload"]) ear.eval() ear_step = int(ck.get("step") or 0) log(f"aligning with {ear_store.run_id} at step {ear_step}") out_path = align_path(ear_store.run_id, ear_step) limit = cfg.get("align_limit", 0) or len(lj) data = {} if out_path.exists(): with gzip.open(out_path, "rt") as f: data = json.load(f) log(f"align cache has {len(data)} clips, extending") done = 0 for i, item in enumerate(lj.items): if i >= limit or item["id"] in data: continue wav, _ = lj.load(i) mel = sp.mel_spectrogram(torch.from_numpy(wav)[None].to(device))[0] with torch.no_grad(): logp = ear(mel[None])[0] phones = g2p.text_to_phones(item["text"]) if not phones: continue assign, dur = sp.ctc_viterbi_align(logp.cpu(), ev.encode(phones), upsample=4, n_mel=mel.shape[1]) data[item["id"]] = {"dur": dur.tolist(), "n_mel": int(mel.shape[1])} done += 1 if done % 500 == 0: store.log(stage="align", clips=len(data), added=done) log(f" aligned {len(data)} clips ({done} this pass)") write_alignments_atomic(out_path, data) store.repo and store.repo.submit(out_path, f"shared/{out_path.name}", "alignments") if cfg.get("minutes") and done and done % 50 == 0: pass write_alignments_atomic(out_path, data) store.save(stage="align", step=ck["step"], payload={"meta": {"clips": len(data)}}, metrics={"clips": len(data)}) if store.repo: store.repo.submit(out_path, f"shared/{out_path.name}", "alignments") store.repo.drain() log(f"alignment complete: {len(data)} clips -> {out_path}") return out_path def load_alignments(store_for_prefix, log=print): """The most recent alignment file -- by *time*, not by name. Names are not a safe order here: they contain run ids and step counts, and plain string order puts "99999" after "128503". The newest file on disk is what the trainer means, so that is what it reads. """ files = sorted(Path("/marimo/runs/shared").glob("align_*.json.gz"), key=lambda p: p.stat().st_mtime) if not files: raise SystemExit("no alignment cache found - run --stage align first") with gzip.open(files[-1], "rt") as f: return json.load(f), files[-1] # ------------------------------------------------------------------ speech def trunk_arch(store, d, vocab): """Architecture of a checkpoint in `store`, from its own `latest/meta.pt`. `latest/model.pt` is a bare state dict with no `meta` key, so the old `ck["payload"].get("meta", {})` was always empty and quietly fell back to the launching config. `meta.pt` is written from the model object itself. """ got = store.load_latest(tag="meta") meta = got["payload"] if got and isinstance(got.get("payload"), dict) else {} return (VLMConfig.from_dict(meta.get("model") or d["model"]), int(meta.get("n_words") or vocab.n_words), int(meta.get("n_phones") or vocab.n_phones)) def stage_speech(cfg, store, log=print, device="cuda"): """Train phone + duration + mel heads on the shared trunk, mixed with vision-language batches so the model stays all-inclusive.""" from omni.vlm import OmniVLM, VLMConfig lj, g2p = lj_and_g2p() d = cfg if d.get("vocab") == "v2": import omni.captions as caps vocab = caps.make_vocab_v2() # real captions + speech + scenes else: vocab = od.make_vocab(d["data_seed"]) align, align_file = load_alignments(store, log) log(f"using alignments from {align_file} ({len(align)} clips)") model, start = None, 0 ck = store.load_latest(tag="model") if d["resume"] != "never" else None if ck and ck.get("payload"): mcfg, n_words, n_phones = trunk_arch(store, d, vocab) if n_words != vocab.n_words: raise RuntimeError( f"{store.run_id} declares {n_words} words but this vocabulary has " f"{vocab.n_words}; resuming would pair the checkpoint with the wrong " f"embedding table. Pass --resume never for a fresh run.") model = OmniVLM(mcfg, n_words=n_words, n_phones=n_phones) model.load_state_dict(ck["payload"]) start = ck["step"] log(f"RESUMED {store.run_id} at step {start} on {n_words} words") else: # Warm start from a trained trunk: the speech heads sit on that same trunk, # so the eyes and the language ability come along. Discover the newest run # of `base_stage` -- its id is a hash of its own config, which this process # cannot recompute, and the old hand-built id paired with `enabled=False` # meant this branch could never download anything and never fired. import omni.hub as hub from omni.train import token_from_env base_stage = d.get("base_stage") or "vlm" base, bstate = hub.open_stage_run(d["hf_repo"], base_stage, token_from_env()) bck = base.load_latest(tag="model") if base else None bn_words = 0 if base is not None: _, bn_words, _ = trunk_arch(base, d, vocab) if bck and bck.get("payload") and bn_words == vocab.n_words: mcfg, n_words, n_phones = trunk_arch(base, d, vocab) model = OmniVLM(mcfg, n_words=n_words, n_phones=n_phones) missing, unexpected = model.load_state_dict(bck["payload"], strict=False) log(f"warm start: trunk from {base.run_id} ({base_stage}) step {bck['step']}, " f"{len(missing)} fresh tensors (the speech heads), " f"{len(unexpected)} dropped") else: why = (f"{base_stage} trunk has {bn_words} words, this vocabulary has " f"{vocab.n_words}" if bn_words else f"no {base_stage} checkpoint found") model = OmniVLM(VLMConfig.from_dict(d["model"]), n_words=vocab.n_words, n_phones=vocab.n_phones) log(f"no compatible trunk to warm start from ({why}) - from scratch") model = model.to(device) opt = torch.optim.AdamW(model.parameters(), lr=d.get("sp_lr") or d.get("lr", 2e-4), weight_decay=0.01, betas=(0.9, 0.95), fused=True) if ck: ok = store.load_latest(tag="optim") if ok and ok.get("payload"): from omni.train_vlm import load_opt_flat load_opt_flat(opt, ok["payload"]) scaler = torch.amp.GradScaler("cuda", enabled=False) amp = torch.autocast("cuda", dtype=torch.bfloat16) _sched, _note = lr_warmup_cosine(store, d["steps"], d.get("warmup", 200)) if _note: log(" " + _note) _base_lr = d.get("sp_lr") or d.get("lr", 2e-4) ds = scenes.SceneDataset(seed=d["data_seed"], px=d["img_size"]) ids = [i for i, it in enumerate(lj.items) if it["id"] in align] log(f"stage=speech clips_with_alignment={len(ids)} batch={d['batch']} steps={d['steps']}") t0 = time.time() step = start w = d.get("sp_loss_weights", {"phone": 1.0, "dur": 0.5, "mel": 1.0}) while step < d["steps"]: use_vl = (np.random.default_rng(step).random() < d.get("vl_mix", 0.25)) if use_vl: sidx = od.batch_indices_for_step(step, d["batch"], d["data_seed"]) spec = scenes.spec_batch(sidx, seed=d["data_seed"], px=d["img_size"]) b = od.collate_specs(vocab, spec, device=device, px=d["img_size"]) with amp: out = model.forward_text(b["ids"], images=b["images"], targets=b["targets"], loss_mask=b["mask"]) loss = out["loss"] extra = {"vl_loss": float(out["loss"].detach())} else: ci = [ids[k] for k in clip_indices_for_step(step, d["batch"], d.get("sp_seed", 1), len(ids), offset=start)] batch = load_batch(lj, ci, max_sec=d.get("sp_max_sec", 8.0), seed=d.get("sp_seed", 1)) keep, wavs, texts, phones, durs = [], [], [], [], [] for wv, tx in batch: ph = g2p.text_to_phones(tx) item = lj.items[ci[len(keep)]] a = align.get(item["id"]) if not a or not ph or len(ph) > 300: continue dur = np.array(a["dur"], np.int64) if dur.size != len(ph): continue keep.append(item["id"]); wavs.append(wv); texts.append(tx) phones.append(ph); durs.append(dur) if len(keep) < 2: step += 1 continue Tp = max(len(p) for p in phones) Tid = max(len(vocab.words_to_ids(textmod.words_of(t))) for t in texts) text_ids = torch.full((len(keep), Tid), vocab.pad, dtype=torch.long) phone_ids = torch.zeros((len(keep), Tp), dtype=torch.long) dur_t = torch.zeros((len(keep), Tp), dtype=torch.long) assign, mels = [], [] for i, (t, ph, dr, wv) in enumerate(zip(texts, phones, durs, wavs)): ti = vocab.words_to_ids(textmod.words_of(t))[:Tid] text_ids[i, :len(ti)] = torch.tensor(ti) phone_ids[i, :len(ph)] = torch.tensor(vocab.phones_to_ids(ph)) dur_t[i, :len(dr)] = torch.tensor(dr) assign.append(np.repeat(np.arange(len(ph)), dr)) mels.append(sp.mel_spectrogram(torch.from_numpy(wv)[None])[0].numpy()) Tm = max(a.size for a in assign) assign_t = torch.zeros((len(keep), Tm), dtype=torch.long) mel_t = torch.full((len(keep), Tm, sp.N_MELS), float(np.log(1e-5))) mel_mask = torch.zeros((len(keep), Tm), dtype=torch.bool) for i, (a, m) in enumerate(zip(assign, mels)): n = min(a.size, m.shape[1], Tm) assign_t[i, :n] = torch.tensor(a[:n]) mel_t[i, :n] = torch.from_numpy(m[:, :n].T) mel_mask[i, :n] = True text_ids, phone_ids, assign_t = text_ids.to(device), phone_ids.to(device), assign_t.to(device) dur_t, mel_t, mel_mask = dur_t.to(device), mel_t.to(device), mel_mask.to(device) with amp: out = model.forward_speech(text_ids, phone_ids, assign_t, mel=mel_t, mask=mel_mask, durations=dur_t) loss = (w["phone"] * out["phone_loss"] + w["dur"] * out["dur_loss"] + w["mel"] * out["mel_loss"]) extra = {"phone_loss": float(out["phone_loss"].detach()), "dur_loss": float(out["dur_loss"].detach()), "mel_loss": float(out["mel_loss"].detach()), "phone_acc": float(out["phone_acc"])} opt.zero_grad(set_to_none=True) scaler.scale(loss).backward() scaler.unscale_(opt) torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) for _g in opt.param_groups: _g["lr"] = _base_lr * _sched(step) scaler.step(opt) scaler.update() step += 1 if step % d.get("log_every", 20) == 0: sps = (time.time() - t0) / max(1, step - start) store.log(stage="speech", step=step, loss=float(loss.detach()), sec_per_step=round(sps, 3), **extra) log(f"speech step {step:6d} loss {float(loss.detach()):.3f} " + " ".join(f"{k}={v:.3f}" for k, v in extra.items())) if store.should_checkpoint(step) or step >= d["steps"]: store.save(stage="speech", step=step, payload={"model": model.state_dict(), "optim": _flat(opt), "meta": {"model": model.cfg.to_dict(), "n_words": vocab.n_words, "n_phones": vocab.n_phones, "kind": "omni"}}, metrics={"loss": float(loss.detach())}) _speech_sample(model, vocab, g2p, lj, ids[0], device, log) if d.get("minutes") and (time.time() - t0) > d["minutes"] * 60: break return model, step @torch.no_grad() def _speech_sample(model, vocab, g2p, lj, clip_i, device, log, n=96): model.eval() txt = lj.items[clip_i]["text"] ids = torch.tensor([vocab.words_to_ids(textmod.words_of(txt)[:40])], device=device) s = model.synth(ids, max_phones=n) log(f" synth[{txt[:48]}] -> {len(s['phones'][0])} phones, mel {tuple(s['mel'].shape)}") model.train() # ----------------------------------------------------------------- vocoder def stage_vocoder(cfg, store, log=print, device="cuda"): lj, _ = lj_and_g2p() d = cfg voc = sp.Vocoder().to(device) opt = torch.optim.AdamW(voc.parameters(), lr=d.get("voc_lr") or d.get("lr", 2e-4), weight_decay=0.01, fused=True) # GAN training is opt-in: it fixes the "flat audio" failure mode of a pure # reconstruction loss, at the cost of a second optimiser and more VRAM. disc = sp.MultiResolutionDiscriminator().to(device) if d.get("adversarial") else None dopt = (torch.optim.AdamW(disc.parameters(), lr=d.get("disc_lr", 2e-4), betas=(0.8, 0.99)) if disc else None) log(f" adversarial={'on' if disc else 'off'}") start = 0 if d["resume"] != "never": ck = store.load_latest(tag="model") if ck and ck.get("payload"): voc.load_state_dict(ck["payload"]) start = ck["step"] ok = store.load_latest(tag="optim") if ok and ok.get("payload"): from omni.train_vlm import load_opt_flat load_opt_flat(opt, ok["payload"]) log(f"RESUMED {store.run_id} at step {start}") log(f"stage=vocoder params={sum(p.numel() for p in voc.parameters())/1e6:.2f}M " f"batch={d['batch']} steps={d['steps']} start={start}") t0 = time.time() step = start losses = [] amp = torch.autocast("cuda", dtype=torch.bfloat16) _sched, _note = lr_warmup_cosine(store, d["steps"], d.get("warmup", 200)) if _note: log(" " + _note) _base_lr = d.get("voc_lr") or d.get("lr", 2e-4) while step < d["steps"]: crop = d.get("voc_crop_sec", 1.0) idxs = clip_indices_for_step(step, d["batch"], d.get("sp_seed", 1), len(lj), offset=start) batch = load_batch(lj, idxs, crop_sec=crop, seed=d.get("sp_seed", 1)) wavs = [w for w, _ in batch if len(w) > sp.SR * 0.2] if len(wavs) < 2: step += 1 continue n = min(len(w) for w in wavs) x = torch.from_numpy(np.stack([w[:n] for w in wavs])).to(device) with torch.no_grad(): mel = sp.mel_spectrogram(x) with amp: y = voc(mel) with torch.autocast("cuda", enabled=False): yf, xf = y.float(), x.float() # the generator emits exactly frames*hop samples while the crop is # whatever the wav had; align them before any spectral comparison n = min(xf.shape[-1], yf.shape[-1]) xf, yf = xf[..., :n], yf[..., :n] loss = sp.stft_loss(yf, xf) if disc is not None: if step % 2 == 0: # discriminator step r_out, r_feat = disc(xf) f_out, f_feat = disc(yf.detach()) dl = sp.discriminator_loss(r_out, f_out) dopt.zero_grad(set_to_none=True) dl.backward() torch.nn.utils.clip_grad_norm_(disc.parameters(), 1.0) dopt.step() disc_loss = float(dl.detach()) else: # generator step f_out, f_feat = disc(yf) with torch.no_grad(): r_out, r_feat = disc(xf) adv = sp.generator_adv_loss(f_out) fm = sp.feature_match_loss(r_feat, f_feat) loss = loss + d.get("adv_weight", 1.0) * adv + d.get("fm_weight", 2.0) * fm opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(voc.parameters(), 1.0) for _g in opt.param_groups: _g["lr"] = _base_lr * _sched(step) opt.step() losses.append(float(loss.detach())) step += 1 if step % d.get("log_every", 20) == 0: sps = (time.time() - t0) / max(1, step - start) store.log(stage="vocoder", step=step, stft_loss=float(np.mean(losses[-d.get("log_every", 20):])), adversarial=bool(disc), sec_per_step=round(sps, 3)) log(f"vocoder step {step:6d} stft {np.mean(losses[-d.get('log_every',20):]):.3f}") if store.should_checkpoint(step) or step >= d["steps"]: store.save(stage="vocoder", step=step, payload={"model": voc.state_dict(), "optim": _flat(opt), "meta": {"kind": "vocoder"}}, metrics={"stft": float(np.mean(losses[-d.get("log_every", 20):]))}) if d.get("minutes") and (time.time() - t0) > d["minutes"] * 60: break return voc, step return @app.cell(hide_code=True) def lib_train(): # ===== omni/train.py ===== module body (imports come from `setup`) sys.path.insert(0, "/marimo") DEFAULTS = dict( preset="large", steps=3000, batch=96, accum=2, lr=6e-4, wd=0.05, warmup=200, clip=1.0, workers=12, img_size=128, amp_dtype="bf16", log_every=10, lookahead=6, push_seconds=1200.0, data_seed=0, channels_last=True, minutes=0.0, ckpt_steps=200, ckpt_seconds=600.0, hf_repo="hkblockchaintechnology/omni-vlm-ckpt", resume="auto", run_tag="", vocab="v1", base_stage="", ) def token_from_env(): tok = os.environ.get("HF_TOKEN") if not tok: f = Path("/marimo/.hf_token") if f.exists(): tok = f.read_text().strip() return tok def identity(cfg: dict) -> dict: """Config that must match for a resume to be legitimate.""" return {k: cfg[k] for k in ("preset", "model", "img_size", "data_seed", "vocab_size", "vocab_hash")} def make_cfg(args) -> dict: cfg = dict(DEFAULTS) for k, v in vars(args).items(): if v is not None and k not in ("config", "stage"): cfg[k] = v if args.config: cfg.update(json.loads(args.config)) for kv in args.set: k, v = kv.split("=", 1) cfg[k] = json.loads(v) import omni.data as od import omni.train_vlm as tvlm if getattr(args, "stage", "") == "captions" or cfg.get("vocab") == "v2": import omni.captions as _caps # the real-data run has its own vocabulary cfg.update(_caps.vocab_identity_v2()) else: cfg.update(od.vocab_identity(cfg["data_seed"])) model = tvlm.preset(cfg["preset"]).to_dict() model["img_size"] = cfg["img_size"] cfg["model"] = model return cfg #: Optimisation settings a resume quietly changes when the launcher does not pass #: them. They are deliberately *not* part of the run identity (a resume should be #: free to change batch size), but the defaults are not the run's history: the #: real-data run trains at lr 3e-4 and `RunConfig` defaults to 4e-4, so a bare #: `omni.cli train --stage captions` used to carry on at the wrong learning rate. DRIFT_KEYS = ("lr", "batch", "accum", "steps", "warmup", "clip") def config_drift(recorded: dict, launch: dict, keys=DRIFT_KEYS) -> dict: """{key: (recorded, launch)} for optimisation settings that disagree.""" return {k: (recorded[k], launch[k]) for k in keys if k in recorded and k in launch and recorded[k] != launch[k]} def _embedding_rows(cfg): """Total rows of the token embedding this config will build, or None. `vocab_size` counts raw words; `CombinedVocab` prefixes its special tokens, so the embedding is that much wider. Used to check a checkpoint against its own meta rather than against the config it was recorded with. """ try: from omni.vlm import CombinedVocab if cfg.get("vocab_size"): return int(cfg["vocab_size"]) + len(CombinedVocab.SPECIAL_WORDS) except Exception: pass return None def get_store(cfg, stage, adopt_existing=True): """RunStore for `stage`, resuming the newest compatible run when there is one. The computed run id is a hash of the serialised config, so it is not a reliable way to *find* a run -- see `hub.find_resumable_run`. When `resume == "never"` (or `adopt_existing` is off) the computed id is used as-is, which is what "start a brand-new run" means. """ import omni.hub as hub ident = {**identity(cfg), "stage": stage} run_id = f"{stage}-{cfg['preset']}" + (f"-{cfg['run_tag']}" if cfg.get("run_tag") else "") \ + f"-{hub.config_hash(ident)[:6]}" state = None if adopt_existing and cfg.get("resume", "auto") != "never": found_id, state = hub.find_resumable_run(cfg["hf_repo"], stage, identity(cfg), token=token_from_env(), log=print, n_words=_embedding_rows(cfg)) if found_id and found_id != run_id: print(f"[omni.train] computed run id {run_id} is not on the Hub; adopting " f"{found_id} (step {state.get('step')}) instead of starting from zero", flush=True) run_id = found_id if found_id and state: drift = config_drift(state.get("config") or {}, cfg) if drift: print("[omni.train] WARNING: this resume changes optimisation settings " "the run was trained with:", flush=True) for k, (was, now) in sorted(drift.items()): print(f" {k}: recorded {was!r} -> launching {now!r}", flush=True) repo = hub.RemoteRepo(cfg["hf_repo"], token=token_from_env(), private=hub.CKPT_PRIVATE) store = hub.RunStore(run_id, repo=repo, cfg=identity(cfg), ckpt_every_steps=cfg["ckpt_steps"], ckpt_every_seconds=cfg["ckpt_seconds"], keep_milestones=cfg.get("keep_milestones", 3), # `milestone_every` counts *checkpoint* intervals, so on a # fast stage the default 5 means a commit every few minutes: # the vocoder at 848 steps/min was pushing a milestone every # 2.9 min, i.e. 20 of the account's 128 hourly commits on its # own. These knobs existed in RunConfig but were never passed # through, so `--set milestone_every=...` did nothing. milestone_every=cfg.get("milestone_every", 5), push_every_seconds=cfg.get("push_seconds", 1200.0)) return store #: Launch settings worth keeping. None of them is part of a run's identity, so #: they are free to change on a resume -- but they should not overwrite the #: configuration the run was *created* under, because that is what run discovery #: reads to decide whether a checkpoint is compatible. LAUNCH_KEYS = ("steps", "batch", "accum", "lr", "warmup", "clip", "ckpt_steps", "ckpt_seconds", "push_seconds", "minutes", "log_every", "workers", "vl_mix", "scene_mix", "max_len", "adversarial") def record_launch(store, cfg): """Record a launch without rewriting the run's own definition. `set_progress(config=...)` used to run on every start, so `config` drifted to whatever the most recent process was launched with -- which is how the real-data run's recorded learning rate became 4e-4 while its weights had been trained at 3e-4. The first writer wins now, and each launch is recorded in `launch` alongside it. """ state = store.state() patch = {} if not state.get("config"): patch["config"] = cfg patch["launch"] = {k: cfg[k] for k in LAUNCH_KEYS if k in cfg} patch["launches"] = int(state.get("launches") or 0) + 1 patch.update(pid=os.getpid(), host=os.uname().nodename, started=time.time()) return store.set_progress(**patch) def attach_signal_handlers(store): def _bye(signum, frame): print(f"\n[signal {signum}] flushing checkpoint state to the hub...", flush=True) try: store.set_progress(interrupted=int(time.time()), signal=int(signum)) # the local run dir is ahead of the uploader queue: push it directly import omni.hub as _hub # the box is going away: this push matters more than the hourly budget _hub.resync_local_runs(store.repo.repo_id if store.repo else "", token=token_from_env(), retries=1, priority=True) finally: os._exit(0) for s in (signal.SIGTERM, signal.SIGINT): try: signal.signal(s, _bye) except Exception: pass def execute(cfg, stage: str, log=print): """Run one stage from a validated RunConfig (CLI and notebook entry point). Thin adapter over the stage registry: it owns the RunStore lifecycle so a stage never has to know how checkpoints are addressed. """ from omni.config import STAGE_NAMES, RunConfig rc = cfg if isinstance(cfg, RunConfig) else RunConfig.from_dict(cfg) if stage not in STAGE_NAMES: raise SystemExit(f"unknown stage {stage!r}; choose from {list(STAGE_NAMES)}") if stage not in STAGES: raise SystemExit(f"unknown stage {stage!r}; have {sorted(STAGES)}") d = rc.to_dict() store = get_store(d, stage) record_launch(store, d) attach_signal_handlers(store) print(f"[omni.train] stage={stage} run_id={store.run_id} log={store.run_dir}/train.log", flush=True) # Refuse to "run" a stage that has nothing to do. A resume whose start step is # already at or past the launcher's `--steps` makes every trainer's # `while step < steps` loop false on entry: the process starts, logs, and exits # "finished in 5s" having done nothing, which reads exactly like success. Say so. _recorded = int((store.state() or {}).get("step") or 0) if _recorded and _recorded >= int(d["steps"]): raise SystemExit( f"nothing to do: run {store.run_id} is at step {_recorded} but --steps is " f"{d['steps']}. Raise --steps above {_recorded} to continue it, or start a " f"new run (a fresh --run_tag) to train longer under a new schedule.") t0 = time.time() STAGES[stage](d, store, log=log) store.set_progress(finished=time.time(), seconds=round(time.time() - t0, 1), stage=stage) if store.repo: store.repo.drain(timeout=300) print(f"[omni.train] stage {stage} finished in {time.time()-t0:.0f}s", flush=True) return 0 def run_vlm(cfg, store, log=print): import omni.data as od from omni.vlm import OmniVLM, VLMConfig vocab = od.make_vocab(cfg["data_seed"], cache=f"/marimo/runs/shared/vocab_words.json") model, start, resume = None, 0, None if cfg["resume"] != "never": ck = store.load_latest(tag="model") if ck and ck.get("payload"): meta = ck["payload"].get("meta", {}) mcfg = VLMConfig.from_dict(meta.get("model", cfg["model"])) model = OmniVLM(mcfg, n_words=meta.get("n_words", vocab.n_words), n_phones=meta.get("n_phones", vocab.n_phones)) model.load_state_dict(ck["payload"]) ok = store.load_latest(tag="optim") resume = {"optim": ok["payload"]} if ok and ok.get("payload") else None start = ck["step"] log(f"RESUMED {store.run_id} at step {start} (from_remote={ck['from_remote']}) " f"stage={ck['stage']}") import omni.train_vlm as tvlm return tvlm.train(cfg, store, vocab, model=model, start_step=start, resume=resume, log=log) def run_ear(cfg, store, log=print): import omni.train_speech as ts return ts.stage_ear(cfg, store, log=log) def run_align(cfg, store, log=print): import omni.train_speech as ts return ts.stage_align(cfg, store, log=log) def run_speech(cfg, store, log=print): import omni.train_speech as ts return ts.stage_speech(cfg, store, log=log) def run_vocoder(cfg, store, log=print): import omni.train_speech as ts return ts.stage_vocoder(cfg, store, log=log) def run_captions(cfg, store, log=print): import omni.captions as caps return caps.stage_captions(cfg, store, log=log) STAGES = {"captions": run_captions, "vlm": run_vlm, "ear": run_ear, "align": run_align, "speech": run_speech, "vocoder": run_vocoder} def train_main(argv=None): ap = argparse.ArgumentParser("omni.train") ap.add_argument("--stage", required=True) ap.add_argument("--steps", type=int) ap.add_argument("--batch", type=int) ap.add_argument("--accum", type=int) ap.add_argument("--lr", type=float) ap.add_argument("--minutes", type=float) ap.add_argument("--preset") ap.add_argument("--img_size", type=int) ap.add_argument("--log_every", type=int) ap.add_argument("--workers", type=int) ap.add_argument("--run_tag", default="") ap.add_argument("--resume", default="auto") ap.add_argument("--config", default="") ap.add_argument("--ckpt_steps", type=int) ap.add_argument("--ckpt_seconds", type=float) ap.add_argument("--data_seed", type=int) ap.add_argument("--hf_repo") ap.add_argument("--set", action="append", default=[], help="extra cfg overrides, key=json") args = ap.parse_args(argv) cfg = make_cfg(args) store = get_store(cfg, args.stage) record_launch(store, cfg) attach_signal_handlers(store) print(f"[omni.train] stage={args.stage} run_id={store.run_id} " f"log={store.run_dir}/train.log", flush=True) if args.stage not in STAGES: raise SystemExit(f"unknown stage {args.stage}; have {sorted(STAGES)}") _recorded = int((store.state() or {}).get("step") or 0) if _recorded and _recorded >= int(cfg["steps"]): raise SystemExit( f"nothing to do: run {store.run_id} is at step {_recorded} but --steps is " f"{cfg['steps']}. Raise --steps above {_recorded}, or start a new run.") t0 = time.time() STAGES[args.stage](cfg, store, log=print) store.set_progress(finished=time.time(), seconds=round(time.time() - t0, 1), stage=args.stage) store.repo and store.repo.drain(timeout=300) print(f"[omni.train] stage {args.stage} finished in {time.time()-t0:.0f}s", flush=True) return @app.cell(hide_code=True) def lib_infer( CombinedVocab, OmniVLM, VLMConfig, avatar, hub, od, scenes, sp, textmod, ): # ===== omni/infer.py ===== module body # Imports come from `setup` (the materializer collects exactly what each body uses); # `materialize` writes this back out to /marimo/omni/infer.py. def vocab_for_rows(rows: int, log=None): """The recorded vocabulary whose size matches an embedding with `rows` rows. A token space is two halves of one thing -- the word list and the embedding -- and this project has two of them: v1 (8 011 words, 8 057 rows with phones) and v2 (16 380 words, 16 426 rows). Mixing them does not crash. It loads perfectly, then answers fluent nonsense, which is how the better model stayed unserved for hours and how a first attempt at this fix scored 0.0% instead of 18.5%. The embedding's row count is ground truth because it cannot disagree with the weights, so the vocabulary is chosen to fit it rather than assumed. Note the count to compare is `len(CombinedVocab(...))`, not the raw word count: seven special tokens are prepended, so 16 380 words are 16 387 word rows. """ for name in ("vocab_words_v2.json", "vocab_words.json"): p = Path("/marimo/runs/shared") / name if not p.exists(): continue try: words = json.loads(p.read_text()) except Exception: continue v = CombinedVocab(words=words, phones=textmod.PHONES) if len(v) == rows: if log: log(f" vocabulary: {v.n_words} words from {name} " f"(matches the {rows}-row embedding)") return v if log: log(f" no recorded vocabulary matches a {rows}-row embedding") return None class OmniSystem: """The trained artefact: one trunk + ear + vocoder.""" #: mel bins of the served trunk; refreshed from the first synthesis so a #: non-default geometry (tests, other configs) needs no globals. n_mels: int = int(sp.N_MELS) def __init__(self, model, ear, vocoder, vocab, g2p, cfg, device="cuda", loaded=None): self.model, self.ear, self.vocoder = model, ear, vocoder self.vocab, self.g2p, self.cfg, self.device = vocab, g2p, cfg, device self.loaded = loaded or {} for m in (self.model, self.ear, self.vocoder): if m is not None: m.eval() # ---------------------------------------------------------------- loading @classmethod def load(cls, cfg: dict, device="cuda", token=None, log=print, require=("vlm",)): from omni.train import identity as run_identity token = token or _token() loaded = {} def _fetch(stage): """What this stage serves: the checkpoint, its meta, and its state.""" store, state = hub.open_stage_run(cfg["hf_repo"], stage, token) if store is None: loaded[stage] = None return None ck = store.load_for_inference(tag="model") mck = store.load_latest(tag="meta") meta = mck.get("payload") if mck and isinstance(mck.get("payload"), dict) else {} loaded[stage] = {"run_id": store.run_id, "step": ck["step"] if ck else 0, "from_remote": bool(ck and ck["from_remote"])} return {"ck": ck, "meta": meta or {}, "state": state or {}} # The trunk decides the token space, so fetch it before building the vocabulary. trunk = None for stage in ("speech", "vlm"): info = _fetch(stage) if info and info["ck"] and info["ck"].get("payload"): trunk = (stage, info) break vocab, model = None, None if trunk: stage, info = trunk payload, meta = info["ck"]["payload"], info["meta"] rows = payload["tok.weight"].shape[0] if "tok.weight" in payload else 0 vocab = vocab_for_rows(rows, log=log) if rows else None if vocab is None: vocab = od.make_vocab(cfg["data_seed"]) ear = voc = None if trunk: stage, info = trunk payload, meta = info["ck"]["payload"], info["meta"] mcfg = VLMConfig.from_dict(meta.get("model") or cfg["model"]) n_words = int(meta.get("n_words") or vocab.n_words) n_phones = int(meta.get("n_phones") or vocab.n_phones) model = OmniVLM(mcfg, n_words=n_words, n_phones=n_phones) model.load_state_dict(payload) model.to(device).eval() log(f"loaded {stage} trunk from {loaded[stage]['run_id']} " f"step {loaded[stage]['step']}") else: log("no trunk checkpoint found - answering will not work") ck = _fetch("ear") if ck and ck["ck"] and ck["ck"].get("payload"): ear = sp.Ear(n_out=len(sp.EarVocab())).to(device) ear.load_state_dict(ck["ck"]["payload"]) ear.eval() log(f"loaded ear from step {loaded['ear']['step']}") ck = _fetch("vocoder") if ck and ck["ck"] and ck["ck"].get("payload"): voc = sp.Vocoder().to(device) voc.load_state_dict(ck["ck"]["payload"]) voc.eval() log(f"loaded vocoder from step {loaded['vocoder']['step']}") g2p = textmod.G2P("/marimo/data/cmudict.dict") return cls(model, ear, voc, vocab, g2p, cfg, device, loaded) # ------------------------------------------------------- public artefacts @classmethod def from_bundle(cls, source="hkblockchaintechnology/VLM-all-inclusive", filename="weights/omni_weights.pt", token=None, device=None, dict_path=None, log=print): """Load the *published* weights bundle. No run dirs, no private repo. `source` may be a Hub repo id or a local directory. This is the path a stranger takes: download the artifact, call this, make a movie. """ import os device = device or ("cuda" if torch.cuda.is_available() else "cpu") token = token or _token() local = Path(source) if local.exists(): bundle_path = local / filename if local.is_dir() else local vocab_file = bundle_path.parent / "vocab_words.json" else: from huggingface_hub import hf_hub_download bundle_path = Path(hf_hub_download(source, filename, token=token)) try: vocab_file = Path(hf_hub_download(source, "weights/vocab_words.json", token=token)) except Exception: vocab_file = Path("/marimo/runs/shared/vocab_words.json") log(f"loading bundle {bundle_path} ({bundle_path.stat().st_size/1e9:.2f} GB)") bundle = torch.load(bundle_path, map_location="cpu", weights_only=False) meta = bundle.get("meta", {}) # Find the trunk first: its embedding decides what the word list must be. A # word list only decodes correctly if it is exactly as wide as the embedding, # and the failure mode is silent -- the model loads and answers fluent # nonsense -- so the sizes are checked rather than trusted. This is what # caught the published bundle, which shipped a v2 trunk with a v1 word list. trunk_sd = next((bundle[n] for n in ("speech", "vlm") if n in bundle and "tok.weight" in bundle[n]), None) rows = int(trunk_sd["tok.weight"].shape[0]) if trunk_sd is not None else None words = meta.get("vocab_words") if words is None and vocab_file.exists(): words = json.loads(Path(vocab_file).read_text()) if words is not None and rows: candidate = CombinedVocab(words=words, phones=textmod.PHONES) if len(candidate) != rows: log(f" the bundle's word list is {len(words)} words but its trunk has " f"{rows} embedding rows; looking for a vocabulary that fits") recorded = vocab_for_rows(rows, log=log) if recorded is None: raise RuntimeError( f"this bundle's trunk has a {rows}-row embedding and no available " f"word list matches it, so its outputs cannot be decoded. Use a " f"bundle whose meta carries `vocab_words`.") words = recorded.words if words is None: raise RuntimeError("the bundle has no vocabulary; pass a bundle that does " "(or place vocab_words.json next to it)") vocab = CombinedVocab(words=words, phones=textmod.PHONES) if rows and len(vocab) != rows: raise RuntimeError( f"vocabulary is {len(vocab)} tokens but the trunk's embedding is {rows}; " f"refusing to serve a model that would answer nonsense") mcfg = VLMConfig.from_dict(meta["model"]) model = OmniVLM(mcfg, n_words=meta.get("n_words", vocab.n_words), n_phones=meta.get("n_phones", vocab.n_phones)) for name, holder in (("vlm", model), ("speech", model)): if name in bundle: holder.load_state_dict(bundle[name]) log(f" trunk: {name} @{meta.get('active', {}).get(name, {}).get('step', '?')}") break ear = voc = None if "ear" in bundle: ear = sp.Ear(n_out=len(sp.EarVocab())) ear.load_state_dict(bundle["ear"]) if "vocoder" in bundle: voc = sp.Vocoder() voc.load_state_dict(bundle["vocoder"]) for m in (model, ear, voc): if m is not None: m.to(device).eval() g2p = textmod.G2P(dict_path or _find_cmudict()) cfg = dict(meta, img_size=meta.get("img_size", mcfg.img_size), preset=meta.get("preset", "xl"), hf_repo=None, run_tag="published", vocab_size=len(words), vocab_hash=meta.get("vocab_hash", "published")) log(f"ready on {device}: trunk + {'ear ' if ear else ''}{'vocoder' if voc else ''}" f"{' (no ear/vocoder in bundle)' if not (ear and voc) else ''}") return cls(model, ear, voc, vocab, g2p, cfg, device, loaded={"from_bundle": str(bundle_path)}) # ------------------------------------------------------------------ eyes @torch.no_grad() def answer(self, image, question, max_new=24, temperature=0.0, log=print): """image (H,W,3 uint8 or a scene dict) + question -> text answer.""" if isinstance(image, dict): from omni.render import render_specs spec = scenes.spec_batch([0], seed=0, px=self.cfg["img_size"]) for k in range(spec["shape"].shape[1]): spec["shape"][0, k] = scenes.SHAPE_ID[image["objects"][k]["shape"]] if k < len(image["objects"]) else 0 spec["valid"][0, k] = 1.0 if k < len(image["objects"]) else 0.0 if k < len(image["objects"]): ob = image["objects"][k] spec["color"][0, k] = np.array(scenes.COLORS[ob["color"]], np.float32) / 255 spec["cx"][0, k], spec["cy"][0, k] = ob["cx"], ob["cy"] spec["rad"] = spec["r"] spec["r"][0, k] = scenes.SIZE_R[ob["size"]] / self.cfg["img_size"] * (1 + 0.12 * ob.get("jitter", 0)) img_t = render_specs(spec, px=self.cfg["img_size"], device=self.device) else: arr = np.asarray(image, np.float32) if arr.ndim == 2: arr = np.stack([arr] * 3, -1) x = torch.from_numpy(arr).permute(2, 0, 1)[None].to(self.device) if x.shape[-1] != self.cfg["img_size"]: x = torch.nn.functional.interpolate(x, size=(self.cfg["img_size"],) * 2, mode="bilinear", align_corners=False) img_t = (x / 127.5 - 1.0).clamp(-1, 1) if arr.max() > 1.5 else (x * 2 - 1).clamp(-1, 1) prefix = [self.vocab.bos] + self.vocab.words_to_ids(textmod.words_of(question)) + [self.vocab.sep] ids = torch.tensor(prefix)[None].to(self.device) out = self.model.generate_text(ids, images=img_t, max_new=max_new, temperature=temperature) return " ".join(self.vocab.ids_to_words(out[0])) # ----------------------------------------------------------------- voice @torch.no_grad() def speak(self, text, max_phones=200, temperature=0.0): """text -> (phonemes, durations, mel, waveforms).""" if self.model is None: raise RuntimeError("no trunk loaded") ids = torch.tensor([self.vocab.words_to_ids(textmod.words_of(text)[:32])]).to(self.device) out = self.model.synth(ids, max_phones=max_phones, temperature=temperature) phones = [self.vocab.ids_to_phones([int(p)]) for p in out["phones"][0]] phones = [p[0] for p in phones if p] raw = [int(d) for d in out["durations"][0][:len(phones)]] durations = self.calibrate_durations(raw) # Rebuild the mel for the *calibrated* timeline: the audio and the lips # must span exactly the same number of frames or they drift apart. timed = torch.tensor([durations], device=self.device) mel = self.model.mel_from_durations(out["hidden"][:, :len(phones)], timed) mel = mel[0].float().cpu().numpy() self.n_mels = int(mel.shape[-1]) # one path for both backends: it normalises the mel orientation, which a # direct griffin_lim call silently gets wrong wav = avatar.griffin_or_vocoder(mel, vocoder=self.vocoder, device=self.device, n_mels=self.n_mels) wav = self.normalise_wav(wav) return {"phones": phones, "durations": durations, "mel": mel, "wav": wav} @staticmethod def calibrate_durations(durations, frames_per_phone=6.5, max_scale=4.0): """Rescale the predicted durations to a natural speaking rate. The duration head is trained on log1p targets and systematically under- predicts; without this the speech is ~3x too fast. Relative timing (what the lips care about) is preserved. """ d = [max(1, int(x)) for x in durations] if not d: return d mean = sum(d) / len(d) scale = min(max_scale, max(0.25, frames_per_phone / max(mean, 1e-6))) return [max(1, int(round(x * scale))) for x in d] @staticmethod def normalise_wav(wav, target_rms=0.08, peak=0.95, max_gain=2000.0): """Bring the synthesised waveform to a usable level without clipping. The gain is whatever reaches `target_rms`, bounded by the headroom to `peak` and by `max_gain` - the last bound only exists so that a near-silent floor (all noise, no speech) is not amplified to full scale. """ w = np.asarray(wav, np.float32) if w.size == 0: return w rms = float(np.sqrt((w ** 2).mean())) peak_now = float(np.abs(w).max()) if rms <= 0.0 or peak_now <= 0.0: return w gain = min(target_rms / rms, peak / peak_now, max_gain) return np.clip(w * gain, -1.0, 1.0) def lexicon_timeline(self, text, base=2.6): """text -> (phonemes, durations in mel frames) using the CMUdict lexicon. A deterministic stand-in for the voice heads: vowels get more frames than consonants, so the mouth opens on vowels and closes on stops. """ phones = self.g2p.text_to_phones(text) or ["S"] durations = [] for i, p in enumerate(phones): frames = base * (1.9 if p in textmod.VOWELS else 1.0) if p in ("M", "B", "P"): frames *= 0.8 durations.append(max(2, int(round(frames)))) return phones, durations # ------------------------------------------------------------------ ear @torch.no_grad() def listen(self, wav, sr=sp.SR): """audio -> text (the model also hears).""" if self.ear is None: raise RuntimeError("no ear loaded") x = torch.as_tensor(np.asarray(wav, np.float32), device=self.device)[None] logp = self.ear(sp.mel_spectrogram(x))[0].argmax(-1).cpu().numpy() ev = sp.EarVocab() phones, prev = [], None for p in logp: s = ev.itos[int(p)] if s != "" and s != prev: phones.append(s) prev = s return phones # ------------------------------------------------------- test -> movie def test_to_movie(self, image, question, out_dir="/marimo/out", name=None, size=384, fps=avatar.FPS, log=print, max_new=24, timeline="model"): """The headline demo: an image and a question go in, a movie comes out.""" t0 = time.time() out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) name = name or f"movie_{int(time.time())}" text = self.answer(image, question, max_new=max_new, log=log) log(f"answer: {text!r} ({time.time()-t0:.2f}s)") if not text.strip(): text = "i am not sure" speech = self.speak(text) if self.model is not None else None mel = None timeline = "lexicon" if speech is None: phones, durations = self.lexicon_timeline(text) else: phones, durations, mel = speech["phones"], speech["durations"], speech["mel"] # the voice heads need training before their output is meaningful; # until then fall back to a lexicon timeline so the lips still track # the actual words instead of random phones. cold = (len(phones) < 2 or float(np.mean(durations)) < 1.5 or float(sum(durations)) < 0.25 * sp.SR / sp.HOP) if cold: log(f"voice heads still cold (mean {float(np.mean(durations)):.2f} " f"frames/phone, {len(phones)} phones) - the lips use the lexicon " f"timeline until the speech stage is trained") phones, durations = self.lexicon_timeline(text) timeline = "lexicon" else: timeline = "model" wav_path = out_dir / f"{name}.wav" mp4_path = out_dir / f"{name}.mp4" wav = speech["wav"] if speech is not None else np.zeros(int(0.3 * sp.SR), np.float32) sp.write_wav(wav_path, avatar.pad_wav_to_timeline(wav)) avatar.render_talking_head(mel, phones, durations, mp4_path, wav_path=wav_path, fps=fps, size=size, log=log, n_mels=self.n_mels) log(f"test->movie done in {time.time()-t0:.2f}s -> {mp4_path}") return {"question": question, "answer": text, "phones": phones, "durations": durations, "mel": mel, "wav": wav_path, "mp4": mp4_path, "timeline": timeline, "seconds": round(time.time() - t0, 2)} def _find_cmudict(): """CMUdict in any of the places it normally lives; the G2P falls back to letter-to-sound rules when it is missing, so this is best-effort.""" for cand in ("/marimo/data/cmudict.dict", "./data/cmudict.dict", str(Path(__file__).resolve().parents[1] / "data" / "cmudict.dict")): if Path(cand).exists(): return cand return None def _token(): import os t = os.environ.get("HF_TOKEN") if not t and Path("/marimo/.hf_token").exists(): t = Path("/marimo/.hf_token").read_text().strip() return t def default_cfg(preset="xl", data_seed=0, img_size=128, run_tag="main", hf_repo="hkblockchaintechnology/omni-vlm-ckpt"): import omni.train_vlm as tvlm vocab = od.make_vocab(data_seed) cfg = dict(preset=preset, data_seed=data_seed, img_size=img_size, run_tag=run_tag, hf_repo=hf_repo, model=tvlm.preset(preset).to_dict()) cfg["model"]["img_size"] = img_size cfg.update(od.vocab_identity(data_seed)) return cfg return @app.cell(hide_code=True) def materialize(): # --- materialize: rebuild the omni/ package from THIS notebook -------------- # The molab sandbox is not a forever thing, but notebook.py is snapshotted. # So the notebook is the source of truth: this cell extracts every `lib_*` cell # back out into /marimo/omni/*.py and imports it. On a fresh sandbox, run the # notebook top-to-bottom and the entire training stack is on disk again. import ast as _ast import importlib as _importlib import sys as _sys import textwrap as _textwrap OMNI_DIR = Path("/marimo/omni") NB_PATH = Path("/marimo/notebook.py") # cross-module imports (stripped from the cells so marimo's dataflow stays clean) MODULE_IMPORTS = { "config": [], "logging_utils": [], "hub": [], "text": [], "scenes": [], "render": [ "import omni.scenes as S" ], "vlm": [], "speech": [ "import omni.text as textmod" ], "data": [ "import omni.scenes as scenes", "import omni.text as textmod" ], "avatar": [ "import omni.speech as sp", "import omni.text as textmod" ], "train_vlm": [ "from omni.vlm import OmniVLM, VLMConfig, count_params", "import omni.data as od", "import omni.scenes as scenes" ], "train_speech": [ "from omni.train import token_from_env", "from omni.vlm import OmniVLM, VLMConfig", "import omni.data as od", "import omni.hub as hub", "import omni.scenes as scenes", "import omni.speech as sp", "import omni.text as textmod" ], "train": [], "supervise": [], "captions": [ "from omni.vlm import CombinedVocab, OmniVLM, VLMConfig", "import omni.data as od", "import omni.scenes as scenes", "import omni.text as textmod" ], "infer": [ "from omni.vlm import CombinedVocab, OmniVLM, VLMConfig", "import omni.avatar as avatar", "import omni.data as od", "import omni.hub as hub", "import omni.scenes as scenes", "import omni.speech as sp", "import omni.text as textmod" ], "doctor": [], "cli": [ "from omni.config import PRESETS, STAGE_NAMES, RunConfig", "from omni.logging_utils import configure_logging, get_logger" ] } # alias -> submodule, so the notebook itself can use the package ALIASES = {"scenes": "scenes", "S": "scenes", "textmod": "text", "od": "data", "sp": "speech", "avatar": "avatar", "hub": "hub", "render": "render"} def _block(node, src): """Statements of an AST block, dedented, without marimo's `return (...)`.""" stmts = list(node.body) while stmts and (isinstance(stmts[-1], _ast.Return) or (isinstance(stmts[-1], _ast.If) and "__name__" in _ast.dump(stmts[-1].test))): stmts = stmts[:-1] if not stmts: return "" first = stmts[0] start = min([d.lineno for d in getattr(first, "decorator_list", [])] + [first.lineno]) lines = src.splitlines()[start - 1: stmts[-1].end_lineno] return _textwrap.dedent("\n".join(lines)) + "\n" def _is_setup_block(node): return (isinstance(node, _ast.With) and getattr(node.items[0].context_expr, "func", None) is not None and getattr(node.items[0].context_expr.func, "attr", "") == "setup") def cell_body(cell_name, src=None): """Source of a notebook cell (a `with app.setup(...)` block for the setup cell, otherwise a named function), minus the `return (...)` marimo appends.""" src = NB_PATH.read_text() if src is None else src tree = _ast.parse(src) if cell_name == "setup": for node in tree.body: if _is_setup_block(node): return _block(node, src) raise RuntimeError("no `app.setup` block in the notebook") for node in tree.body: if isinstance(node, _ast.FunctionDef) and node.name == cell_name: return _block(node, src) raise RuntimeError(f"cell {cell_name!r} not found in {NB_PATH}") #: Entry points that belong only in the generated file, never in a notebook cell. #: A module whose entry point is missing is a *silent* no-op under #: `python -m omni.` -- which is exactly how the launcher scripts start every #: stage -- so `_verify_modules` checks these on every materialize. MODULE_EPILOGUE = { "supervise": '\nif __name__ == "__main__":\n raise SystemExit(supervisor_main())\n', "train": '\nif __name__ == "__main__":\n raise SystemExit(train_main())\n', "doctor": '\nif __name__ == "__main__":\n raise SystemExit(doctor_main())\n', "cli": '\nif __name__ == "__main__":\n raise SystemExit(main())\n', } def _source_stamp(path=None): """(mtime_ns, sha256) of the notebook file -- this build's *input*. `materialize` generates the package by parsing notebook.py, so a file that changes mid-read produces a package that silently disagrees with the notebook. That is how the real-data v2 stage was lost once: it was written on disk, published, and then overwritten by a materialize that had read a stale notebook. Cheap to detect, so it is detected. """ p = Path(path or NB_PATH) return p.stat().st_mtime_ns, hashlib.sha256(p.read_bytes()).hexdigest() def _verify_modules(omni_dir=None): """Loudly report a generated module that is missing a piece of itself. `write_text` cannot fail on a truncated table -- it just writes less -- and a module without its `__main__` block imports perfectly and does nothing at all. That is a silent, total failure of `python -m omni.cli train`, which is how every launcher starts a stage, so the generated package is checked each time. """ omni_dir = Path(omni_dir or OMNI_DIR) problems: list[str] = [] for mod in MODULE_IMPORTS: path = omni_dir / f"{mod}.py" if not path.exists(): problems.append(f"{mod}.py was not written") continue text = path.read_text() try: compile(text, str(path), "exec") except SyntaxError as exc: problems.append(f"{mod}.py does not compile: {exc}") ep = MODULE_EPILOGUE.get(mod) if ep: call = ep.strip().splitlines()[-1].strip() if call not in text: problems.append(f"{mod}.py is missing `{call}`, so " f"`python -m omni.{mod}` would silently do nothing") return problems #: Bare names a module body may use -> the import that provides them. The #: setup cell is the pool, but a minimal per-module header cannot regenerate #: the pool by itself; this table closes that loop. NAME_IMPORTS = { "Any": "from typing import Any", "Iterable": "from typing import Iterable", "Mapping": "from typing import Mapping", "Sequence": "from typing import Sequence", "Callable": "from typing import Callable", "Optional": "from typing import Optional", "nn": "from torch import nn", "F": "import torch.nn.functional as F", "torch": "import torch", "np": "import numpy as np", "Image": "from PIL import Image", "ImageDraw": "from PIL import ImageDraw", "ImageFilter": "from PIL import ImageFilter", "json": "import json", "math": "import math", "os": "import os", "re": "import re", "sys": "import sys", "time": "import time", "logging": "import logging", "argparse": "import argparse", "hashlib": "import hashlib", "shutil": "import shutil", "importlib": "import importlib", "signal": "import signal", "gzip": "import gzip", "atexit": "import atexit", "queue": "import queue", "threading": "import threading", "zipfile": "import zipfile", "tempfile": "import tempfile", "io": "import io", "random": "import random", "subprocess": "import subprocess", "Path": "from pathlib import Path", "asdict": "from dataclasses import asdict", "dataclass": "from dataclasses import dataclass", "field": "from dataclasses import field", "fields": "from dataclasses import fields", "replace": "from dataclasses import replace" } def _bound_names(statement: str) -> list[str]: """Names an import statement binds (``import x.y as z`` -> ``z``).""" try: tree = _ast.parse(statement) except SyntaxError: return [] names: list[str] = [] for node in tree.body: if isinstance(node, _ast.Import): names += [(a.asname or a.name.split(".")[0]) for a in node.names] elif isinstance(node, _ast.ImportFrom): names += [(a.asname or a.name) for a in node.names] return names def _write_atomic(path, text: str) -> None: """Write a generated module without ever exposing a half-written file. The training workers restart on crash and import these modules, and `Path.write_text` truncates before it writes -- so a worker that restarts in that window imports a torn file and dies on a SyntaxError it cannot explain. tmp + `os.replace` is atomic within a filesystem. """ path = Path(path) tmp = path.with_suffix(path.suffix + ".tmp") tmp.write_text(text) os.replace(tmp, path) def module_header(header: str, body: str) -> str: """Give a module only the imports it uses. The setup cell is the union of every dependency in the project; writing all of it into twelve modules would leave ~250 unused imports behind (and a linter that cannot see genuine mistakes through the noise). """ used = {n.id for n in _ast.walk(_ast.parse(body)) if isinstance(n, _ast.Name)} kept: list[str] = [] for line in header.splitlines(): stripped = line.strip() if not stripped or stripped.startswith("#") or "__future__" in stripped: kept.append(line) continue names = _bound_names(stripped) if not names or any(n in used for n in names): kept.append(line) return "\n".join(kept) + "\n" def materialize(write=True, verbose=False): """lib_* cells -> /marimo/omni/*.py, then import them.""" before = _source_stamp() src = NB_PATH.read_text() header = cell_body("setup", src) # the setup cell is the import pool pool = {line.strip() for line in header.splitlines() if line.strip()} for mod in MODULE_IMPORTS: # complete it from the bodies for name in {n.id for n in _ast.walk(_ast.parse(cell_body(f"lib_{mod}", src))) if isinstance(n, _ast.Name)}: if name in NAME_IMPORTS: pool.add(NAME_IMPORTS[name]) header = "\n".join(sorted(pool, key=lambda l: (not l.startswith("from __future__"), l))) + "\n" mods = {} for mod, imports in MODULE_IMPORTS.items(): body = cell_body(f"lib_{mod}", src) if write: _write_atomic(OMNI_DIR / f"{mod}.py", module_header(header, body) + "".join(i + "\n" for i in imports) + body + MODULE_EPILOGUE.get(mod, "")) mods[mod] = body for mod in MODULE_IMPORTS: name = f"omni.{mod}" if name in _sys.modules: _importlib.reload(_sys.modules[name]) else: _importlib.import_module(name) if write: problems = _verify_modules() if problems: raise RuntimeError("materialize wrote an incomplete omni/ package:\n " + "\n ".join(problems)) if _source_stamp() != before: raise RuntimeError( "notebook.py changed while materialize was reading it, and it is " "this cell's build input -- so /marimo/omni may not match the " "notebook. Nothing is lost: run this cell again now that the " "editor has saved.") return mods materialized = materialize() scenes = _importlib.import_module("omni.scenes") S = scenes textmod = _importlib.import_module("omni.text") od = _importlib.import_module("omni.data") sp = _importlib.import_module("omni.speech") avatar = _importlib.import_module("omni.avatar") hub = _importlib.import_module("omni.hub") render = _importlib.import_module("omni.render") vlm = _importlib.import_module("omni.vlm") infer = _importlib.import_module("omni.infer") print("materialized:", ", ".join(materialized)) print("omni/ files:", ", ".join(sorted(p.name for p in OMNI_DIR.glob("*.py")))) print("stage entry points:", ", ".join(_importlib.import_module("omni.train").STAGES)) return S, avatar, hub, infer, od, scenes, sp, textmod @app.cell(hide_code=True) def launch(CFG, HF_TOKEN, LOG_DIR, RUN_ROOT, hub): # --- Run training: detached, resumable, mirrored to the Hub ----------------- # A marimo cell must not block for hours, and the process should outlive a # kernel restart. So training runs detached and checkpoints every # ckpt_steps / ckpt_seconds. Press Start again any time: it RESUMES. STAGE_HELP = { "vlm": "eyes + brain: image/question -> answer (needs nothing)", "ear": "the Ear: audio -> phonemes (CTC) (needs the LJSpeech corpus)", "align": "forced-align LJSpeech with the Ear -> durations (cached + uploaded)", "speech": "voice: text -> phonemes/durations/mel, mixed with the VLM task", "vocoder": "mel -> waveform, from scratch (multi-resolution STFT loss)", } def launch(stage, steps=None, batch=None, extra="", tag=None, print_cmd=True): """Start or resume `stage` in a detached process. Returns the command.""" tag = tag or CFG["run_tag"] args = dict(steps=steps or CFG["steps"], batch=batch or CFG["batch"], accum=CFG["accum"], lr=CFG["lr"], log_every=CFG["log_every"], ckpt_steps=CFG["ckpt_steps"], ckpt_seconds=CFG["ckpt_seconds"], data_seed=CFG["data_seed"], img_size=CFG["img_size"], preset=CFG["preset"]) flags = " ".join(f"--{k} {v}" for k, v in args.items()) cmd = (f"cd /marimo && setsid nohup env HF_TOKEN=$(cat /marimo/.hf_token) " f"HF_HUB_DISABLE_PROGRESS_BARS=1 python -m omni.train --stage {stage} {flags} " f"--run_tag {tag} {extra} > {LOG_DIR}/{stage}.log 2>&1 < /dev/null &") subprocess.run(cmd, shell=True, executable="/bin/bash") if print_cmd: print(cmd) return cmd def running(): """Which training processes are alive in this sandbox.""" out = subprocess.run("pgrep -af 'python -m omni.train' || true", shell=True, capture_output=True, text=True).stdout rows = [] for line in out.strip().splitlines(): if "--stage" in line: parts = line.split() stage = line.split("--stage")[1].split()[0] rows.append({"pid": parts[0], "stage": stage}) return rows #: The Hub table is not free: one `list_repo_files` plus a `read_json` per run. #: The monitor cell re-runs on a timer, so an uncached version spends the API #: quota merely by being *open* -- a dashboard should never be what gets an #: account rate limited. Cache it; the table carries its own "read at" stamp. _HUB_STATUS: dict = {"t": 0.0, "rows": []} def hub_status(max_age: float = 300.0, force: bool = False): """What the Hub holds per stage - i.e. the resume point after a wipe. Reuses a read younger than `max_age` seconds. `force=True` always re-reads. """ if not force and _HUB_STATUS["rows"] and (time.time() - _HUB_STATUS["t"]) < max_age: return _HUB_STATUS["rows"] repo = hub.RemoteRepo(CFG["hf_repo"], token=HF_TOKEN) files = repo.api().list_repo_files(CFG["hf_repo"]) if repo.enabled else [] rows = [] for path in files: if path.endswith("meta/state.json"): run_id = path.split("/")[1] st = repo.read_json(path) or {} rows.append({"stage": st.get("stage", run_id.split("-")[0]), "run": run_id, "step": st.get("step"), "loss": round(float(st.get("loss", 0)), 4), "updated": time.strftime("%H:%M:%S", time.localtime(st.get("updated_at", 0)))}) rows = sorted(rows, key=lambda r: (r["stage"], r["run"])) _HUB_STATUS.update(t=time.time(), rows=rows) return rows def hub_read_age() -> float | None: """Seconds since the Hub table was actually read (None if never).""" return None if not _HUB_STATUS["t"] else time.time() - _HUB_STATUS["t"] def backup_state() -> dict: """The periodic pusher's heartbeat, plus what is still only on this box. Both halves are *local* file reads, so this is free and can be refreshed as often as the notebook likes -- unlike `hub_status`, which talks to the Hub. """ beat = {} p = Path(LOG_DIR) / "backup_daemon.json" if p.exists(): try: beat = json.loads(p.read_text()) except Exception: beat = {} rows = [] for d in sorted(Path(RUN_ROOT).glob("*")): sf = d / "meta" / "state.json" if not sf.exists() or d.name == "shared": continue try: st = json.loads(sf.read_text()) except Exception: continue ck = d / "latest" / "model.pt" age = time.time() - ck.stat().st_mtime if ck.exists() else None rows.append({"run": d.name, "stage": st.get("stage", d.name.split("-")[0]), "local_step": st.get("step"), "ckpt_age": age, "has_optim": (d / "latest" / "optim.pt").exists()}) rows.sort(key=lambda r: (str(r["stage"]), r["run"])) return {"beat": beat, "local": rows} def served_table(force_hub: bool = False) -> list[dict]: """One row per run: local step, the Hub's step, and how far behind it is.""" hub_by_run = {r["run"]: r for r in hub_status(force=force_hub)} out = [] for r in backup_state()["local"]: h = hub_by_run.get(r["run"], {}) local, hub = r["local_step"], h.get("step") age = r["ckpt_age"] out.append({ "stage": r["stage"], "run": r["run"], "local": local, "hub": hub, "behind": (local - hub) if isinstance(local, int) and isinstance(hub, int) else None, "last ckpt": f"{age/60:.1f}m" if isinstance(age, (int, float)) else "-", "optim": r["has_optim"], "hub seen": h.get("updated", "-")}) return out return ( STAGE_HELP, backup_state, hub_read_age, hub_status, launch, running, served_table, ) @app.cell(hide_code=True) def monitor(LOG_DIR, RUN_ROOT, backup_state, hub_read_age, served_table): # --- Monitor: live local logs + the Hub state a fresh sandbox would resume from import altair as alt import pandas as pd refresh = mo.ui.refresh(options=["30s", "2m", "10m"], default_interval="2m") def read_runs(): frames = [] for d in sorted(Path(RUN_ROOT).glob("*")): log = d / "log.jsonl" if not log.exists(): continue rows = [] for line in log.read_text().splitlines()[-4000:]: try: rows.append(json.loads(line)) except Exception: pass if rows: df = pd.DataFrame(rows) df["run"] = d.name frames.append(df) return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(columns=["run", "step", "loss"]) def metric_chart(df, y, title): """A line per run. Keeps only the columns the chart names. Altair serialises the whole frame it is handed, and these rows come from seven different stages with different metric keys, so most cells of most columns are NaN -- and NaN is not valid JSON, so handing over the full frame raises "Out of range float values are not JSON compliant". Project first, then draw. """ if df.empty or y not in df: return mo.md(f"_{title}: no data yet_") keep = [c for c in ("run", "step", y) if c in df] d = df[keep].dropna() if d.empty: return mo.md(f"_{title}: no data yet_") return alt.Chart(d).mark_line().encode( x=alt.X("step:Q", title="step"), y=alt.Y(f"{y}:Q", title=title), color=alt.Color("run:N", title="run"), tooltip=["run", "step", y], ).properties(height=170, title=title) _metrics = read_runs() _logs = sorted(Path(LOG_DIR).glob("*.log"), key=lambda p: p.stat().st_mtime) _beat = backup_state()["beat"] _rows = served_table() _age = hub_read_age() if _beat: _err = _beat.get("last_error") _status = (f"**periodic pusher:** every {_beat.get('interval', '?')}s, last cycle " f"{_beat.get('updated', '?')} ({_beat.get('seconds', '?')}s), pushed " f"{', '.join(_beat.get('pushed') or []) or 'nothing'}" + (f" - **last error: {_err}**" if _err else "")) else: _status = ("**periodic pusher:** not running - start it with " "`nohup python backup_daemon.py --interval 1200 &` so a dead box costs " "minutes of work instead of hours") monitor = mo.vstack([ mo.md("## Monitor"), refresh, mo.md(_status), mo.md(f"_Hub table read {_age:.0f}s ago; it is cached, so this dashboard does not " f"spend the API quota while it is open._" if _age is not None else "_Hub table: not read yet._"), mo.ui.table(_rows or [{"stage": "-", "run": "no local runs yet"}], selection=None, label="this box vs the Hub ('behind' = steps not yet pushed)"), mo.hstack([metric_chart(_metrics, "loss", "text / speech loss"), metric_chart(_metrics, "acc", "answer accuracy")], gap=1), mo.hstack([metric_chart(_metrics, "ctc_loss", "ear CTC"), metric_chart(_metrics, "stft_loss", "vocoder STFT")], gap=1), mo.accordion({"tail of the most recent training log": mo.plain_text(_logs[-1].read_text()[-3000:] if _logs else "none")}), ]) return (pd,) @app.cell(hide_code=True) def about(): # --- The project front page --------------------------------------------------- # Same structure as the Hub card: what it is, then the roadmap and status. The # figures are the ones `card_figures` generates, so the notebook and the published # card cannot disagree about what the model looks like or how far it has run. about = mo.vstack([ mo.md(r""" # OmniVLM — *VLM-all-inclusive* **One trunk. One token space. Every modality is a head on it.** The same weights read an image, choose the words, time the phonemes, synthesise the waveform *and* drive a face — so the face cannot disagree with the voice it came from. Trained from **random initialisation** on one RTX PRO 6000: no pretrained vision backbone, no pretrained TTS, no HiFi-GAN, no external forced aligner. | part | what it is | trained here | |---|---|---| | **eyes** | ViT patch encoder | yes — on procedurally rendered scenes | | **brain** | causal trunk, cross-attends the vision tokens | yes | | **text** | word head: question → answer | yes | | **ear** | CTC phoneme recogniser over log-mel; also produces the alignments | yes — on LJSpeech | | **voice** | phoneme + duration heads + FastSpeech-style mel decoder, same trunk | yes | | **sound** | Vocos-style mel → waveform | yes — no HiFi-GAN | | **lips** | viseme parameters → a photoreal face whose mouth is a generated shape per viseme | shapes generated once; the timing is the model's | """), mo.hstack([mo.image("/marimo/repo/assets/hero.png", width=330), mo.vstack([mo.image("/marimo/repo/assets/roadmap.png", width=520), mo.md("_Live status and the roadmap. Regenerate with the " "`card_figures` cell._")])], gap=1, align="start"), ]) return @app.cell def plain_english(LOG_DIR, RUN_ROOT): # --- In plain words: what this is, and whether it is working --------------- # Everything else in this notebook is machinery. This cell is the dashboard, in # sentences, for a human who did not write any of it. It reads only *local* files, # so it is instant and costs nothing to refresh. # # step one batch of practice. More steps = more practice. # loss a mistake score. Lower is better; it should drift downwards. # checkpoint a save file. Training restarts from the newest one. # the Hub the cloud copy of those save files. If this box dies, the work is # still there and training carries on from it. PLAIN_STAGES = { "vlm": ("Eyes + brain", "looks at a picture and answers a question"), "captions": ("Real photographs", "learns what real things actually look like"), "speech": ("Voice", "decides how each word should be said"), "ear": ("Ear", "turns sound into the sounds of speech"), "vocoder": ("Sound", "turns the plan into an audible waveform"), } #: Which key each stage writes to its *log* (the dashboard reads logs, but #: `hub.BEST_METRIC` names the keys used at save time, which are shorter). PLAIN_LOG_KEY = {"vlm": "loss", "captions": "loss", "speech": "loss", "ear": "ctc_loss", "vocoder": "stft_loss"} def plain_status(limit=2500): """One friendly row per job: how far along, which way it is moving, is it saved.""" beat = {} p = Path(LOG_DIR) / "backup_daemon.json" if p.exists(): try: beat = json.loads(p.read_text()) except Exception: beat = {} rows = [] for stage, (title, doing) in PLAIN_STAGES.items(): newest, best = None, None for d in sorted(Path(RUN_ROOT).glob(f"{stage}-*")): lg = d / "log.jsonl" if not lg.exists() or lg.stat().st_size == 0: continue age = time.time() - lg.stat().st_mtime if newest is None or age < newest[0]: newest = (age, d, lg) if newest is None: rows.append({"job": f"{title} ({stage})", "what it does": doing, "progress": "not started", "moving": "-", "saved": "-"}) continue age, d, lg = newest recs = [] for line in lg.read_text(errors="replace").splitlines()[-limit:]: try: recs.append(json.loads(line)) except Exception: pass key = PLAIN_LOG_KEY.get(stage, "loss") pairs = [(r.get("step"), r[key]) for r in recs if isinstance(r.get(key), (int, float)) and isinstance(r.get("step"), int)] step = pairs[-1][0] if pairs else (recs[-1].get("step") if recs else 0) moving = "not enough data yet" if len(pairs) >= 40: tenth = max(1, len(pairs) // 10) recent = sum(v for _, v in pairs[-tenth:]) / tenth older = sum(v for _, v in pairs[-2 * tenth:-tenth]) / tenth if key == "acc": moving = ("answers right more often" if recent > older * 1.02 else "holding steady") else: moving = ("still improving" if recent < older * 0.98 else "basically flat" if recent < older * 1.02 else "getting WORSE") ck = d / "latest" / "model.pt" saved = (f"{time.time() - ck.stat().st_mtime:.0f}s ago" if ck.exists() else "-") rows.append({"job": f"{title} ({stage})", "what it does": doing, "progress": f"{step:,} steps", "moving": moving, "saved": ("up to date" if age < 180 else f"paused {age/60:.0f} min")}) return rows, beat _plain_rows, _beat = plain_status() _live = [r for r in _plain_rows if r["saved"].startswith(("up to date", "paused"))] _worse = [r["job"] for r in _plain_rows if "WORSE" in r["moving"]] if not _live: _verdict = ("Nothing is training right now. Open the **Train** cell below and press " "**start / resume** — every job picks up from its last save file.") elif _worse: _verdict = ("Training is running, but these jobs are getting *worse* rather than " "better: " + ", ".join(_worse) + ". That is worth a look before leaving " "it overnight.") else: _verdict = ("Good: everything below is training and its save files are being copied " "to the cloud. You can leave it alone.") plain_english = mo.vstack([ mo.md("## Is it working? (plain English)"), mo.md(_verdict), mo.ui.table(_plain_rows, selection=None, label="the five parts of the model, and what each one is doing"), mo.md( "**How to read this**\n\n" "* **progress** is how much practice a part has had. More is not automatically " "better — a score that has stopped falling means it has learned what it can.\n" "* **moving** compares this part's recent scores with the ones just before. " "*still improving* is what you want; *basically flat* means the extra practice " "is buying very little; *getting WORSE* means something needs changing.\n" "* **saved** is how recently a save file was written here. Every save file is " "also copied to the cloud, so the box dying costs minutes, not weeks.\n\n" f"**Cloud copy:** the last full safety sweep was at " f"`{_beat.get('updated', 'never')}` and it uploaded " f"{', '.join(_beat.get('pushed') or []) or 'nothing'}" + (f". ⚠️ *Last error: {_beat['last_error']}*" if _beat.get("last_error") else ".") + "\n\n" "**If you only ever read one cell, read this one.** It answers: is it on, is it " "learning, and is the work safe." ), ]) return @app.cell(hide_code=True) def resume_policy(): # --- Resume policy, roadmap and status --------------------------------------- # The sandbox is ephemeral; only `notebook.py` is snapshotted. This cell is the # operational half of the front page: where the work stands, and what a fresh # session must do. `handoff` above is the detail; this is the summary. resume_policy = mo.vstack([ mo.md(r""" ## Roadmap and status ![roadmap](/marimo/repo/assets/roadmap.png) """), mo.md(r""" | stage | what it does | state | |---|---|---| | **vlm** | eyes + trunk from random weights — the only true pretraining | **done**, 12 000 steps (150% of plan), loss 0.062 | | **ear** | audio → phonemes (CTC), and the alignments | running, 63% — ctc 0.0067 | | **speech** | voice: phonemes, durations, mel | running, 54% | | **vocoder** | mel → waveform | running, 72% — stft 0.866 | | **captions** | real photographs + captions | running, **5%** — the budget needs re-thinking, not more hours | | **align** | forced alignment of the speech corpus | complete, 13 100 clips | **299 distinct checkpoints** archived on the Hub, **873 000 trained steps**, 5 live stages. ## Resume policy — the sandbox is not a forever thing 1. **This notebook is the source of truth.** `materialize` writes every `lib_*` cell back out to `/marimo/omni/*.py`, so a brand-new sandbox rebuilds the stack by running the notebook. 2. **The Hub is the archive** — checkpoints, *every* milestone ever pushed, the ops toolchain, the card assets, the face sprites. Local pruning deletes local copies; the Hub keeps them. That is how the measured-best checkpoint was recovered after pruning deleted it. 3. **Every stage checkpoints to the Hub** (pointer written last, so its presence implies the payload arrived). Local writes are atomic. 4. **Training runs detached**, and installs SIGTERM handlers that flush a final checkpoint. 5. **Resume is exact**: model + optimizer + step + config live in the checkpoint, and batches are a pure function of the step counter. 6. **Run identity is a hash of the config**, and **discovery beats recomputing an id** — a computed hash can name a run that does not exist, which used to look like "no checkpoint" and silently train from zero. 7. **Cost of losing the box** = work since the last commit. A periodic pusher runs every 20 minutes and `final_flush.py` makes a priority flush before the scheduled relaunch. """), ]) return @app.cell(hide_code=True) def challenges(): # --- The challenge bank, rehearsed -------------------------------------------- # Everything here will be attacked, and the attacks are predictable. `docs/CHALLENGES.md` # holds 100 of them, each with the honest answer and — the part that matters — whether the # evidence actually exists. `✓` measured, `~` partial, `✗` not demonstrated. # # The point is not the answers, it is the ratio: of 100 predictable attacks, about 11 are # answered with a measurement worth defending, about 30 with a measured number that is *bad* # and published anyway, and about 25 are open admissions. The evidence plan at the end of # that document is therefore the training design, in priority order. challenge_bank = mo.vstack([ mo.md(r""" ## Rehearsed challenges | | count | meaning | |---|---|---| | **answered with evidence** | ~11 | a measurement I would defend against a hostile reviewer | | **answered with a bad number** | ~30 | measured, unflattering, published anyway | | **open admissions** | ~25 | no evidence, stated as such | | remainder | ~34 | partial, or a definitional argument rather than an empirical one | **[Full bank: `docs/CHALLENGES.md`](../repo/docs/CHALLENGES.md)** — also on the Hub, and summarised in the model card's *Questions we expect* section. ### The six actions the bank dictates 1. **Scheduled sampling** for the phone/duration/mel heads, then re-run `validate_voice.py` until WER < 0.4 — the voice is currently unintelligible (**WER 0.97**). 2. **Freeze the trunk during the speech stage** — more speech training made VQA *worse* (47.0% → 35.5%). 3. **Put the WER test in the eval loop with a hard gate** — the only test here that can falsify a claim. 4. **Confidence intervals and 3 seeds** — nothing is reported with variance. 5. **Random-init and majority-class baselines** on the same 200 questions. 6. **A human rating sheet** for the face and the audio. ### The two facts every metric missed * the voice does not work — `phone_acc 1.000` was measuring a **teacher-forced** objective; * more training made question-answering worse — the loss fell while VQA fell with it. """), ]) return @app.cell(hide_code=True) def movie_player( OUT_DIR, SYSTEMS, fresh_scene, generate_btn, get_system, prompt_box, prompt_image, prompt_seed, prompt_size, reload_btn, scenes, ): # ============================================================================ # CELL 2 of 2 - play the movie (picture + sound together) # ============================================================================ if reload_btn.value: SYSTEMS.clear() get_system() print("weights reloaded:", get_system().loaded) WORKS = {"result": None, "error": None, "image": None} if generate_btn.value: try: sys_ = get_system() if prompt_image.value: image = np.asarray(Image.open(io.BytesIO(prompt_image.value[0].contents)).convert("RGB")) question, truth = prompt_box.value or "what do you see", None else: scene, q_auto, truth = fresh_scene(prompt_seed.value) image = scenes.render_scene(scene, px=252) question = prompt_box.value or q_auto r = sys_.test_to_movie(image, question, out_dir=OUT_DIR, name=f"movie_{prompt_seed.value}", size=int(prompt_size.value)) r["truth"] = truth WORKS.update(result=r, image=image) except Exception: import traceback as _tb WORKS["error"] = _tb.format_exc() # Always show the newest clip, so there is something to play immediately _shown = WORKS["result"] if _shown is None and WORKS["error"] is None: _have = sorted(Path(OUT_DIR).glob("*.mp4"), key=lambda f: f.stat().st_mtime) if _have: _mp4 = _have[-1] _wav = _mp4.with_suffix(".wav") _shown = {"mp4": str(_mp4), "wav": str(_wav), "answer": "(press generate for a new one)", "question": "previous run", "truth": None, "seconds": None, "phones": [], "durations": []} _blocks = [mo.md("## 2. Movie (with sound)")] if WORKS["error"]: _blocks.append(mo.callout(mo.md(f"```\n{WORKS['error'][-1200:]}\n```"), kind="danger")) elif _shown: if WORKS["image"] is not None: _blocks.append(mo.image(WORKS["image"], width=220)) _blocks.append(mo.md(f"**prompt** {_shown['question']} \n**answer** {_shown['answer']}" + (f" \n_ground truth: {_shown['truth']}_" if _shown.get("truth") else ""))) _blocks.append(mo.video(src=str(_shown["mp4"]))) # picture + sound _blocks.append(mo.audio(src=str(_shown["wav"]))) # the same take, audio only _blocks.append(mo.hstack([ mo.download(str(_shown["mp4"]), label="download the mp4"), mo.download(str(_shown["wav"]), label="download the wav"), ], gap=1)) if _shown.get("seconds"): _blocks.append(mo.md(f"_{len(_shown['phones'])} phonemes · " f"{sum(_shown['durations'])} mel frames · " f"{_shown['seconds']}s end to end_")) else: _blocks.append(mo.md("_press **generate video with sound** in the cell above_")) movie_player = mo.vstack(_blocks) return @app.cell(hide_code=True) def model_card(CFG, infer): # --- The model card (published as the repo README) --------------------------- MODEL_CARD_TEMPLATE = r"""--- library_name: omni tags: [vision-language, tts, lipsync, multimodal, from-scratch, molab] license: apache-2.0 ---
# OmniVLM ### One model that sees, answers, speaks — and moves its lips. **One trunk. One token space. Every modality is a head on it.** ![trained from scratch](https://img.shields.io/badge/trained%20from%20scratch-no%20pretrained%20parts-4F46E5?style=flat-square) ![stages](https://img.shields.io/badge/stages-5%20on%20one%20GPU-0D9488?style=flat-square) ![exact match](https://img.shields.io/badge/VQA%20exact%20match-47.0%25-C2831B?style=flat-square) ![checkpoints](https://img.shields.io/badge/checkpoints-299%20archived-39404B?style=flat-square) ![licence](https://img.shields.io/badge/licence-Apache--2.0-6B7280?style=flat-square) ![python](https://img.shields.io/badge/python-3.13-3776AB?style=flat-square)
Trained from **random initialisation** on a single RTX PRO 6000 — no pretrained vision backbone, no pretrained TTS, no HiFi-GAN, no external forced aligner. The same weights read an image, choose the words, time the phonemes, synthesise the waveform *and* drive a face, so the face cannot disagree with the voice it came from. ![the whole pipeline](assets/pipeline.png) --- ## Status and roadmap ![roadmap](assets/roadmap.png) | stage | what it does | state | |---|---|---| | **vlm** | eyes + trunk, from random weights — the only true pretraining | **done**, 12 000 steps (150% of plan), loss 0.062 | | **ear** | audio → phonemes (CTC); also produces the alignments | running, 63% — ctc 0.0067 | | **speech** | voice: phonemes, durations, mel, on the same trunk | running, 54% | | **vocoder** | mel → waveform, from scratch | running, 72% — stft 0.866 | | **captions** | real photographs + human captions on that trunk | running, **5%** — needs its budget re-thought | | **align** | forced alignment of the speech corpus | complete, 13 100 clips | **299 distinct checkpoints** are archived on the Hub, across 5 live stages, from **873 000 trained steps**. Every stage checkpoints to the Hub and resumes exactly; the sandbox is ephemeral by design, not by accident. ## Try it ```bash pip install torch numpy pillow scipy huggingface_hub && apt-get install -y ffmpeg git clone https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive python scripts/make_movie.py --question "describe the image" ``` One image and one question in; an mp4 with the answer spoken and the mouth moving comes out — and the published [samples](samples/) carry their own audio track, so picture and voice stay in sync when you play them. --- ## What it is Text-to-image, text-to-speech and text-to-video are separate models that each re-learn language. Here there is **one token space and one trunk**: the same weights that read an image and answer a question also drive the speech heads and the lips. The speech stages are trained *multi-task with* the vision-language task, so the trunk keeps both abilities instead of being forked into per-task copies. | part | implementation | trained here | |---|---|---| | **eyes** | ViT patch encoder | yes — on procedurally rendered scenes | | **brain** | causal trunk, cross-attends the vision tokens | yes | | **text** | word head: question → answer | yes | | **ear** | CTC phoneme recogniser over log-mel, and it produces the alignments too | yes — on the public-domain LJSpeech corpus | | **voice** | phoneme + duration heads + FastSpeech-style mel decoder, on the *same* trunk | yes | | **sound** | Vocos-style mel → waveform | yes — multi-resolution STFT loss, no HiFi-GAN | | **lips** | ARPAbet → viseme parameters → a photoreal face whose *mouth* is composited from a generated shape per viseme, cross-faded between | the shapes were generated once; the timing that selects them is the model's own phoneme timeline | ![one trunk, heads are the modalities](assets/architecture.png) *Everything enters and leaves through one trunk. The lips read the same phoneme timeline that produced the audio, so the face cannot disagree with the voice.* ![the lips, on the phoneme timeline](assets/visemes.png) *The ten viseme mouth shapes, and the mouth saying one real sentence in real time.*
From scratch — and you can check it Every weight here was initialised from random numbers and trained on this machine. There is no pretrained backbone underneath, and no borrowed network anywhere in the pipeline: the vision encoder, the trunk, the phoneme recogniser, the duration and mel heads and the vocoder are all created empty and taught from the corpora. That is a rare thing to be able to say, so it should not be taken on trust — it is a claim you can falsify in one command: ```bash grep -rEn 'from_pretrained|torch[.]hub|(from|import) (transformers|timm|torchvision)|whisper|wav2vec|hifigan' omni/ notebook.py # no matches # no matches ``` The complete list of things this project downloads from the internet is **three datasets and one dictionary** — no models: | downloaded | what it is | licence | |---|---|---| | LJSpeech-1.1 | 24 h of read speech, one speaker — the audio corpus | public domain | | flickr30k | 31 783 photographs, 5 human captions each | CC-BY 4.0 | | CMUdict | an ARPAbet **pronunciation dictionary**, 135 166 entries: text in, phonemes out | BSD-2 | | — | the scene images are *not* downloaded: `omni/scenes.py` generates them | — | Two footnotes, because "from scratch" should not be allowed to do quiet work: * **CMUdict is a lexicon, not a model.** It is a list of how English words are pronounced. A word outside it falls back to ten hand-written letter-to-sound mappings in `omni/text.py` (`a -> AH`, `x -> K S`, ...) — not a learned grapheme-to-phoneme network. * **The viseme table is hand-authored, not learned.** `omni/avatar.py` maps 39 ARPAbet phones onto 15 mouth shapes using numbers a person chose. What the *model* contributes is the timing — its own phoneme sequence and its own durations — and what the renderer does with that is arithmetic and a rasteriser. So the honest summary is: a new architecture, new weights, trained from random initialisation on public data, with the whole pipeline published. If a future version of this repository ever loads a pretrained component, the grep above will stop returning nothing. That is the point of stating it as a check rather than as a claim.
## Why an all-inclusive model — the ground for OmniVLM **Today's multimodal models are, almost always, one output modality per model.** Look at what actually ships on the Hub — sampled by pipeline tag, first 1 000 results each, via `https://huggingface.co/api/models?filter=`, **17 Nov 2025**: | pipeline tag | models (sampled) | what it gives you | |---|---|---| | `image-text-to-text` | >1 000 | sees, answers in text — **no sound, no face** | | `text-to-speech` | >1 000 | says it — **no face, no visual grounding** | | `text-to-video` / `image-to-video` | >1 000 each | moves pixels — **the voice is bolted on afterwards, with no shared representation** | | `any-to-any` | >1 000 | the all-inclusive bucket — but it stops at **audio out** (e.g. `Qwen/Qwen2.5-Omni-7B`, `openbmb/MiniCPM-o-2_6`); no rendered, lip-synced face | | `audio-to-audio`, `text-to-audio` | >1 000 each | voice/audio-only stacks | And lip sync, the piece that makes any of it look alive, is a **niche of ~200 repos** outside every mainstream pipeline: `lipsync` 40, `wav2lip` 54, `sadtalker` 58, `musetalk` 51, `talking-head` 15 — there is **no Hub pipeline tag for it at all**, and nearly all of those are weights for a handful of upstream projects that each want a *pre-existing video plus a pre-existing audio track*. So the field's default shape is three half-measures: 1. **video with no sound** — a diffusion video model; you add TTS later, and the mouth was never conditioned on the words it is now speaking; 2. **sound with no video** — an excellent TTS model; no face, and no grounding in anything visual; 3. **TTS with no lip sync** — or lips driven by a *second*, separately trained model that has never seen the first one's representation of language. Each seam is a place where the modalities disagree. Bolting them together cannot fix that, because there is no shared token space in which "the word I am saying", "the sound I am making" and "the shape my mouth is in" are the same object. **That is the ground OmniVLM stands on.** Here the modalities are not stitched: there is one token space (words **and** ARPAbet phones in one vocabulary) and one causal trunk whose heads are the modalities — vision tokens cross-attended for grounding, a word head, a phoneme head, a duration head and a mel decoder on the *same* hidden state, with the speech stages trained multi-task alongside the vision-language task so neither ability is forked off. The lips are therefore driven by **the model's own phoneme timeline** — the same timeline that produced the waveform — instead of by a separate lip-sync network. And unlike the `any-to-any` giants, the whole thing is trained from scratch on one GPU with published checkpoints, a public held-out metric (`BLEU-4` on flickr30k val) and scripts that reproduce it. **Stated fairly:** the claim is *not* that no multimodal model exists — Gemini/Qwen-Omni-class systems are more capable than this, and `any-to-any` is a real, thriving category. The claim is narrower and checkable: **no published artefact trains eyes, language, voice, sound and lips jointly, from scratch, and ships the whole pipeline — weights, eval, resume policy — as one thing you can download and retrain.** That is the gap this project exists to fill, and it is why the quality caveats below are the honest frontier rather than a hedge. ## What this is for The unusual property here -- one token space, one trunk, every modality trained together -- is exactly what an *agent* needs, because an agent has to see, decide, speak and be seen doing it in the same moment, not in a pipeline of separate models: | where it applies | why this shape fits | |---|---| | **computer-use agents** | the agent reads a screen, acts, and narrates what it is doing. Because the seeing and the saying come from one forward pass over one shared representation, the narration cannot drift away from the action it is describing. | | **phone / on-device agents** | one small trunk plus three tiny heads replaces a separate vision model, a separate ASR, a separate TTS and a separate lip-sync stack. Fewer artefacts to ship, one thing to keep aligned, and the whole assistant can live on the device instead of making a cloud round trip per modality. | | **robotics and embodied agents** | a robot that perceives, answers and speaks from one set of weights is a robot whose words match its perception. And because the mouth is driven by the *same* phoneme timeline that produced the audio, the face and the voice are generated together rather than dubbed over one another. | ![three models, three clocks — or one trunk, one clock](assets/simultaneity.png) The field is full of models that are brilliant at one channel and hand the rest to a different model with a different vocabulary, a different training run and a different clock. Every seam between them is latency, and every seam is somewhere the channels can disagree. ### Emotion is a property of simultaneity, not a fourth model Feeling is not a separate head bolted on at the end; it is what the *coincidence* of the channels conveys. A mouth one phoneme behind its voice reads as uncanny. A voice that says one thing while the eyes and brow say another reads as insincere. For a face to move a person, the timing has to be real -- one decision, at one instant, driving the word, the sound and the movement. That is what training the modalities together buys: the timings that carry feeling are computed from a single hidden state rather than reconstructed afterwards by an integrator guessing at alignment. It is what lets a delivery carry not only knowledge and a machine voice, but something that reads as intention and warmth -- smooth, aligned and coherent from the inside out, whether it arrives through a desktop agent, a phone, or a robot standing in front of you. **Stated honestly:** on *this* checkpoint the claim is architectural, not measured. These weights were trained on procedurally rendered shapes and one speaker's read speech, so what ships today is a small research artefact. The point of the table above is that the architecture has somewhere to go that a stitched pipeline cannot follow -- and the checkpoints, the eval and the resume policy are published so that someone can take it there. ## Questions we expect — and honest answers Full list: **[100 rehearsed challenges with their evidence status](docs/CHALLENGES.md)**.
The twelve that will be asked first **"Your voice is gibberish."** — Yes. An independent Whisper transcribes the synthesised speech at **mean WER 0.97**, and each sentence runs 15 s where 2 s is expected: the decoder loops instead of stopping. The cause is teacher forcing. **"But your metrics said `phone_acc 1.000`."** — And that is the lesson. The phoneme head is trained to predict each phone *given the true previous phone*, so it never learns to run on its own output. **A teacher-forced loss is not evidence about generation.** The card reports the WER because the loss could not have told you. **"More training made it worse?"** — Yes, and it is measured: VQA exact match **47.0%** at step 39 002 versus **35.5%** at 49 502. The speech stage fine-tunes the whole trunk on a speech objective, which erodes the vision-language ability. Serving is pinned to the better checkpoint. **"From scratch — really?"** — Check it. A grep for `from_pretrained`, `torch.hub`, `transformers`, `timm`, `torchvision`, `whisper`, `wav2vec` and `hifigan` across all 20 source files returns nothing. Three datasets and one pronunciation *dictionary* are the only downloads. **"The face is generated, so the demo is not from scratch."** — Correct, and stated: the portrait is SDXL-generated once and its mouth is ten generated shapes selected by the model's own phoneme timeline. Only the timing is the model's. **"You judged the face with a model, not a human."** — Yes — and that model's aesthetic bar is photorealism, so it cannot judge a deliberately stylised face. A human rating sheet is in the evidence plan. **"Everything is self-referential."** — It was. The WER test with Whisper is the first thing in this project that can falsify a claim, and it immediately did. **"No baseline, no variance, no ablation."** — Correct on all three. Single seeds, no random-init baseline, nothing removed and re-measured. **"Your learning rate reached zero."** — It did: the vocoder sat at `stft` 0.9130 for 30 000 steps at `lr = 0.000e+00`. Now floored at 5%. **"You pruned your own best checkpoints."** — Twice. Recovered from the Hub, and pruning now protects every pin. **"Your backup fails silently."** — It did, three ways: a private-storage ceiling that refused every commit while training continued, an upload that returned a commit URL having uploaded nothing, and a batched commit that was refused with nothing saved. All three are now detected and named. **"Thousands will attack this."** — Yes. Of 100 rehearsed challenges: **~11 are answered with a measurement I would defend, ~30 with a measured number that is bad and published anyway, and ~25 are open admissions.** The engineering claims are hard; the modelling claims are soft.
## Training state on this machine (RTX PRO 6000 Blackwell, 100 GB) {results} ![measured training curves](assets/metrics.png) *Loss per stage, read straight out of each run's `log.jsonl` on this machine.*
Data — three datasets and one lexicon, nothing pretrained * **vision-language**: procedurally rendered scenes (shapes, colours, sizes, 3x3 positions) and grounded Q/A — counting, colour, shape, location, size, captioning, comparatives, left/above/below relations, yes/no. Rendered **on the GPU** with a vectorised rasteriser, which is what keeps the GPU busy instead of waiting on a CPU data loader. * **speech**: LJSpeech-1.1 (public domain, LibriVox, 24 h, single speaker), phonemised with CMUdict; alignments come from the model's own CTC ear, not an external aligner. * One shared vocabulary from *both* corpora ({vocab_size} words) plus the 39-symbol ARPAbet phone set — the shared token space is what lets one trunk serve every modality. ![what it is trained on](assets/scenes.png) *Scenes are generated, so every answer is verifiable — which is what makes the exact-match numbers on this card mean something.*
Resume policy — the sandbox is ephemeral by design The training sandbox is ephemeral; only the notebook is snapshotted. So every stage checkpoints model+optimizer+step to `{hf_repo}` (pointer file written last, so its presence implies the payload arrived), training runs detached so it survives a kernel restart, batches are a pure function of the step counter so a resume continues *exactly* where it stopped, and the run id is a hash of the architecture config so a changed config starts a new run instead of corrupting an old one.
Reproduce — the exact commands ```bash python -m omni.train --stage vlm --preset xl --steps 8000 --batch 384 python -m omni.train --stage ear --steps 4000 --batch 24 python -m omni.train --stage align python -m omni.train --stage speech --steps 6000 --batch 24 python -m omni.train --stage vocoder --steps 4000 --batch 16 ```
## Limitations (honest list) * The vision-language corpus is synthetic: the grounding is real, the world is shapes. Swap `omni/scenes.py` for a real caption/VQA corpus to grow it. * **The voice is not intelligible yet, and the training metrics cannot see it.** An independent judge (`openai/whisper-small`, which has never seen this project) transcribes the synthesised speech at a **mean word error rate of 0.97** over five sentences, and each "sentence" runs 15 s where 2 s is expected — the decoder loops instead of stopping. The internal numbers said the opposite: `phone_acc 1.000`, `dur_loss 0.005`, falling `mel_loss`. **They are teacher-forced**: the phoneme head is trained to predict each phone *given the true previous phone*, so it never learns to run on its own output, and autoregressive synthesis then drifts into repetition. Loss on a teacher-forced objective is not evidence about generation — that is the single most important thing this card can tell you. * **The face is a still image, and the mouth is a set of generated shapes.** A photoreal portrait carries ten mouth states, one per viseme, each produced by img2img from that same portrait at a fixed seed and composited through a feathered mask -- so the identity is pixel-identical outside the mouth and cannot drift. The model's own phoneme timeline selects and cross-fades between them. What that means honestly: the lips articulate through ten shapes rather than through a learned, continuous generator, the head does not turn, and there is no blinking. It is a puppet with the model's timing, not a learned talking-head model. * No safety tuning, no RLHF, no moderation. This is a from-scratch research artefact. """ def model_card(stats, extra=None): cfg = infer.default_cfg(preset=CFG["preset"], data_seed=CFG["data_seed"], img_size=CFG["img_size"], hf_repo=CFG["hf_repo"]) m = cfg["model"] rows = [] for st in ("vlm", "ear", "align", "speech", "vocoder"): s = stats.get(st) if not s: rows.append(f"| {st} | not run yet | |") continue metric = f"loss {s['loss']}" if s.get("loss") else "" rows.append(f"| {st} | `{s.get('run')}` | step {s.get('step')} - {metric} |") results = "| stage | run | progress |\n|---|---|---|\n" + "\n".join(rows) if extra: results += "\n\n" + extra return MODEL_CARD_TEMPLATE.format( patch=m["patch"], img_size=CFG["img_size"], vision_layers=m["vision_layers"], n_layers=m["n_layers"], d_model=m["d_model"], vocab_size=CFG.get("vocab_size", "-"), results=results, hf_repo=CFG["hf_repo"]) return (model_card,) @app.cell def card_figures(RUN_ROOT, avatar, scenes, sp, textmod): # --- Figures for the model card --------------------------------------------- # The card is read by strangers on the Hub, and the figures are most of what they # actually see. These are generated from the project's own modules -- the real scenes # the trunk trains on, the real viseme renderer, the real metric logs -- so the card # cannot drift from the code. # # Every figure shares one visual system: Inter (the weights are installed as separate # font families, so bold text has to ask for them by name), a restrained editorial # palette, rounded surfaces with soft shadows, and no default matplotlib chrome. CARD_ASSETS = Path("/marimo/repo/assets") # ---- palette ----------------------------------------------------------------- _INK = "#0E1116" # headings _BODY = "#39404B" # body text _MUTED = "#8A93A3" # captions, axes _RULE = "#E6E9EF" # hairlines _SURF = "#FFFFFF" _SOFT = "#F5F7FA" _ACCENT = "#4F46E5" # indigo: the model itself _WARM = "#E11D48" # rose: the voice and the lips _COOL = "#0D9488" # teal: the ear, the input side _GOLD = "#C2831B" # amber: data # ---- type -------------------------------------------------------------------- _F = "Inter" _F_MED = "Inter Medium" _F_SEMI = "Inter SemiBold" _F_DISPLAY = "Inter Display SemiBold" _F_ITAL = "Inter" # Inter has no italic here; captions stay upright def _fp(family=_F, size=10): from matplotlib.font_manager import FontProperties return FontProperties(family=family, size=size) def _plt(): """matplotlib on a headless backend, styled once.""" import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt plt.rcParams.update({ "font.family": _F, "font.size": 10.5, "text.color": _BODY, "axes.labelcolor": _BODY, "axes.titlecolor": _INK, "xtick.color": _MUTED, "ytick.color": _MUTED, "xtick.labelsize": 9, "ytick.labelsize": 9, "axes.edgecolor": _RULE, "axes.linewidth": 0.9, "axes.spines.top": False, "axes.spines.right": False, "grid.color": _RULE, "grid.linewidth": 0.8, "figure.facecolor": _SURF, "axes.facecolor": _SURF, "savefig.facecolor": _SURF, "legend.frameon": False, "legend.fontsize": 9, "axes.grid": False, }) return plt def _blank(ax): ax.set_xlim(0, 1) ax.set_ylim(0, 1) ax.axis("off") def _eyebrow(ax, text, x=0.0, y=1.0, color=_MUTED): """A small uppercase kicker, the way an editorial page opens a section.""" ax.text(x, y, " ".join(text.upper()), fontproperties=_fp(_F_SEMI, 8.5), color=color, va="bottom", ha="left") def _h1(ax, text, x=0.0, y=1.0): ax.text(x, y, text, fontproperties=_fp(_F_DISPLAY, 17), color=_INK, va="bottom", ha="left") def _card(ax, x, y, w, h, text, fc=_SURF, ec=_RULE, tc=_BODY, size=10.5, weight=_F_MED, radius=0.030, shadow=True, lw=1.0): """A rounded surface with a soft drop shadow -- prose in a shape, not a box.""" from matplotlib.patches import FancyBboxPatch if shadow: for k, a in ((1.0, 0.030), (0.6, 0.035)): ax.add_patch(FancyBboxPatch( (x + 0.004 * k, y - 0.006 * k), w, h, boxstyle=f"round,pad=0,rounding_size={radius}", linewidth=0, facecolor=_INK, alpha=a, zorder=1)) ax.add_patch(FancyBboxPatch( (x, y), w, h, boxstyle=f"round,pad=0,rounding_size={radius}", linewidth=lw, edgecolor=ec, facecolor=fc, zorder=2)) ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontproperties=_fp(weight, size), color=tc, linespacing=1.55, zorder=3) def _arrow(ax, start, end, color=_MUTED, lw=1.3, rad=0.0, style="-|>", ms=11): from matplotlib.patches import FancyArrowPatch ax.add_patch(FancyArrowPatch(start, end, arrowstyle=style, mutation_scale=ms, linewidth=lw, color=color, zorder=1, connectionstyle=f"arc3,rad={rad}", shrinkA=2, shrinkB=2)) def _caption(fig, text, y=-0.02, size=9.5): fig.text(0.5, y, text, ha="center", fontproperties=_fp(_F, size), color=_MUTED) def _paste(img, layer): """Composite a PIL layer onto the float numpy image (RGB or RGBA).""" from PIL import Image as _I base = _I.fromarray(np.clip(img, 0, 255).astype(np.uint8)).convert("RGBA") base.alpha_composite(layer if layer.mode == "RGBA" else layer.convert("RGBA")) return np.asarray(base.convert("RGB"), np.float32) def _paste_masked(img, colour_img, mask): """Composite `colour_img` only where `mask` is set, with a soft edge.""" layer = colour_img.convert("RGBA") layer.putalpha(mask) return _paste(img, layer) def _face(frame, size): return _crop(frame, size, _FACE_CROP) def _mouth(frame, size): return _crop(frame, size, _MOUTH_CROP) #: Read off the *rendered* frame: with the sprite face the mouth is centred at #: y ~0.716 (measured as where an open and a closed mouth differ most), and the head #: fills x 0.10..0.90. _FACE_CROP = (0.100, 0.045, 0.900, 1.000) _MOUTH_CROP = (0.50, 0.716, 0.100) def _crop(frame, size, box): from PIL import Image as _I im = _I.fromarray(frame) if len(box) == 4: x0, y0, x1, y1 = box px = (int(x0 * size), int(y0 * size), int(x1 * size), int(y1 * size)) else: cx, cy, half = box px = (int((cx - half) * size), int((cy - half) * size), int((cx + half) * size), int((cy + half) * size)) return im.crop(px) def _rounded(im, radius_frac=0.06): """Round an image's corners, so a grid of frames does not look like a contact sheet.""" from PIL import Image as _I, ImageDraw as _D im = im.convert("RGB") w, h = im.size r = int(min(w, h) * radius_frac) mask = _I.new("L", (w, h), 0) _D.Draw(mask).rounded_rectangle([0, 0, w - 1, h - 1], radius=r, fill=255) out = _I.new("RGB", (w, h), _SURF) out.paste(im, (0, 0), mask) return out def _fig_pipeline(path): """The hero: what goes in, what comes out, and that it also runs backwards.""" plt = _plt() fig, ax = plt.subplots(figsize=(11.2, 4.5), dpi=170) _blank(ax) _eyebrow(ax, "one trunk, one token space", 0.012, 0.995) _h1(ax, "Every modality is a head on the same model", 0.012, 0.905) y1, h = 0.455, 0.28 w1, xs = 0.205, [0.030, 0.256, 0.482, 0.708] top = [("image\n+ question", _SURF, _BODY, _F_MED), ("eyes: ViT\n+ causal trunk", "#EEF0FE", _ACCENT, _F_SEMI), ("answer text", _SURF, _BODY, _F_MED), ("phonemes\n+ durations", _SURF, _BODY, _F_MED)] for (label, fc, tc, wt), x in zip(top, xs): _card(ax, x, y1, w1, h, label, fc=fc, tc=tc, weight=wt, size=11) for i in range(3): _arrow(ax, (xs[i] + w1 + 0.006, y1 + h / 2), (xs[i + 1] - 0.006, y1 + h / 2)) _arrow(ax, (xs[3] + w1 / 2, y1 - 0.008), (xs[3] + w1 / 2, 0.395), color=_ACCENT, lw=1.5) y2 = 0.095 bot = [("mel", _SURF, _BODY, _F_MED), ("vocoder\n-> waveform", _SURF, _BODY, _F_MED), ("lips: ARPAbet -> visemes\n-> frames -> mp4", "#FDF0F3", _WARM, _F_SEMI)] for (label, fc, tc, wt), x in zip(bot, [0.708, 0.482, 0.256]): _card(ax, x, y2, w1, h, label, fc=fc, tc=tc, weight=wt, size=11) _arrow(ax, (0.708 + w1 / 2, y2 + h + 0.008), (0.708 + w1 / 2, 0.44), color=_ACCENT, lw=1.5) for a, b in ((0.708, 0.482), (0.482, 0.256)): _arrow(ax, (a - 0.006, y2 + h / 2), (b + w1 + 0.006, y2 + h / 2)) _card(ax, 0.030, y2, w1, h, "audio -> ear (CTC)\n-> phonemes -> text", fc="#EAF7F5", tc=_COOL, weight=_F_SEMI, size=11) ax.text(0.030, 0.045, "and the same weights run backwards", fontproperties=_fp(_F, 9), color=_COOL, va="bottom") ax.text(0.708, 0.045, "trained from scratch: no pretrained backbone,\nno HiFi-GAN, " "no external aligner", fontproperties=_fp(_F, 9), color=_MUTED, va="bottom") fig.savefig(path, bbox_inches="tight", pad_inches=0.30) plt.close(fig) return path def _fig_architecture(path): """Where each modality enters and leaves one trunk.""" plt = _plt() fig, ax = plt.subplots(figsize=(11.2, 4.7), dpi=170) _blank(ax) _eyebrow(ax, "architecture", 0.012, 0.995) _h1(ax, "One trunk; the heads are the modalities", 0.012, 0.905) _card(ax, 0.030, 0.400, 0.120, 0.165, "image", size=11) _card(ax, 0.178, 0.400, 0.120, 0.165, "ViT\npatch 8", size=11) _arrow(ax, (0.152, 0.482), (0.176, 0.482)) _card(ax, 0.350, 0.265, 0.230, 0.455, "causal trunk\n\n16 layers\nd_model 1024\nRoPE", fc="#EEF0FE", tc=_ACCENT, weight=_F_SEMI, size=11.5) _arrow(ax, (0.300, 0.482), (0.348, 0.500), color=_COOL, lw=1.4) ax.text(0.322, 0.620, "vision tokens\ncross-attend", fontproperties=_fp(_F, 9), color=_COOL, ha="center", linespacing=1.5) _card(ax, 0.350, 0.030, 0.230, 0.170, "words + ARPAbet phones\none embedding table\n8 011 + 44", size=10) _arrow(ax, (0.465, 0.204), (0.465, 0.261), color=_ACCENT, lw=1.4) heads = [("word head", "the answer", _SURF, _BODY), ("phoneme head", "what it says", _SURF, _BODY), ("duration head", "how long each phone lasts", _SURF, _BODY), ("mel decoder -> vocoder", "the sound", "#FDF0F3", _WARM)] ys = [0.610, 0.465, 0.320, 0.175] for (title, sub, fc, tc), y in zip(heads, ys): _card(ax, 0.648, y, 0.330, 0.115, "", fc=fc) ax.text(0.668, y + 0.072, title, fontproperties=_fp(_F_SEMI, 11), color=tc, va="center", zorder=4) ax.text(0.668, y + 0.038, sub, fontproperties=_fp(_F, 9.5), color=_MUTED, va="center", zorder=4) _arrow(ax, (0.584, 0.492), (0.646, y + 0.058), color=_RULE, lw=1.1) ax.text(0.648, 0.105, "the lips read the phoneme timeline from the same hidden\n" "state, so the face cannot disagree with the voice", fontproperties=_fp(_F, 9), color=_WARM, va="top", linespacing=1.5) ax.text(0.030, 0.300, "every weight\ninitialised\nat random", fontproperties=_fp(_F, 9), color=_MUTED, va="top", linespacing=1.55) fig.savefig(path, bbox_inches="tight", pad_inches=0.30) plt.close(fig) return path def _fig_scenes(path, n=8, seed=7, px=128): """The actual corpus: rendered scenes with the answers the generator knows are true.""" plt = _plt() rng = np.random.default_rng(seed) fig = plt.figure(figsize=(11.2, 5.4), dpi=150) qtypes = ["count", "color", "shape", "where", "size", "total", "compare", "caption"] gs = fig.add_gridspec(2, 4, hspace=0.55, wspace=0.14, left=0.02, right=0.98, top=0.86, bottom=0.03) for k, qt in enumerate(qtypes): ax = fig.add_subplot(gs[k // 4, k % 4]) scene = scenes.make_scene(rng) q, a = scenes.question_answer(scene, rng, [qt]) ax.imshow(_rounded(Image.fromarray(scenes.render_scene(scene, px=px)), 0.055), interpolation="nearest") ax.set_xticks([]); ax.set_yticks([]) for s in ax.spines.values(): s.set_visible(False) ax.set_title(f"\u201c{q}\u201d", fontproperties=_fp(_F_MED, 8.6), color=_BODY, pad=6) ax.text(0.5, -0.10, a, transform=ax.transAxes, ha="center", va="top", fontproperties=_fp(_F_SEMI, 9), color=_ACCENT) fig.text(0.02, 0.965, "SCENES ARE GENERATED, SO EVERY ANSWER IS VERIFIABLE", fontproperties=_fp(_F_SEMI, 8.5), color=_MUTED) fig.text(0.02, 0.915, "31 k real photographs + procedurally rendered scenes", fontproperties=_fp(_F_DISPLAY, 16), color=_INK) fig.savefig(path, bbox_inches="tight", pad_inches=0.22) plt.close(fig) return path def _fig_hero(path, size=560): """The card's opening image: the character, large, with a one-line statement.""" plt = _plt() fig = plt.figure(figsize=(11.2, 5.0), dpi=170) ax = fig.add_axes([0.0, 0.0, 0.47, 1.0]) ax.axis("off") frame = Image.fromarray(avatar.render_frame((0.16, 0.10, 0.05, 0.30), size=size, t=0.35, energy=0.5)) ax.imshow(_rounded(frame, 0.045), interpolation="lanczos") tx = 0.52 fig.text(tx, 0.86, "OMNIVLM", fontproperties=_fp(_F_SEMI, 10), color=_ACCENT) fig.text(tx, 0.72, "It sees,\nanswers, speaks\nand moves its lips", fontproperties=_fp(_F_DISPLAY, 25), color=_INK, linespacing=1.25, va="top") fig.text(tx, 0.36, "One transformer trunk trained from scratch.\n" "The eyes, the words, the voice and the\nmouth are heads on the same model —\n" "so the face cannot disagree with the voice.", fontproperties=_fp(_F, 11.5), color=_BODY, linespacing=1.65, va="top") fig.text(tx, 0.095, "RENDERED BY THIS MODEL'S OWN PHONEME TIMELINE", fontproperties=_fp(_F_SEMI, 8.5), color=_MUTED) fig.savefig(path, bbox_inches="tight", pad_inches=0.22) plt.close(fig) return path def _fig_visemes(path, sentence="the red circle is on the left", size=288, frames=10): """Mouth shapes, and one real sentence moving through them.""" plt = _plt() g2p = textmod.G2P("/marimo/data/cmudict.dict") phones = g2p.text_to_phones(sentence) durations = [max(2, int(round(2.6 * (1.9 if p in textmod.VOWELS else 1.0)))) for p in phones] params, _keys = avatar.viseme_params(phones, durations) names = ["A", "E", "I", "O", "U", "ER", "M", "F", "S", "T"] fig = plt.figure(figsize=(11.2, 5.0), dpi=160) gs = fig.add_gridspec(2, len(names), height_ratios=[1.5, 1.0], hspace=0.30, wspace=0.10, left=0.015, right=0.985, top=0.84, bottom=0.06) for j, name in enumerate(names): ax = fig.add_subplot(gs[0, j]) v = avatar.VISEME_IDS[name] face = Image.fromarray(avatar.render_frame( [v["open"], v["wide"], v["round"], v["press"]], size=size, energy=0.5)) # the mouth itself: with a photoreal face the *shape* is the mouth, and a grid # of whole faces hides exactly the thing this figure exists to show ax.imshow(_mouth(np.asarray(face), face.size[0]), interpolation="lanczos") ax.set_xticks([]); ax.set_yticks([]) for s in ax.spines.values(): s.set_visible(False) ax.set_title(name, fontproperties=_fp(_F_SEMI, 10), color=_WARM if name in ("M", "F") else _INK, pad=5) idx = np.linspace(0, params.shape[0] - 1, frames).astype(int) for j, f in enumerate(idx): ax = fig.add_subplot(gs[1, j]) ax.imshow(np.asarray(_mouth(avatar.render_frame( params[f], size=size, t=f / 25.0, energy=0.6), size))) ax.set_xticks([]); ax.set_yticks([]) for s in ax.spines.values(): s.set_color(_RULE); s.set_linewidth(0.8) ax.set_xlabel(f"{f * sp.HOP / sp.SR:.2f}s", fontproperties=_fp(_F, 8), color=_MUTED) fig.text(0.015, 0.965, "THE LIPS ARE NOT A SECOND MODEL", fontproperties=_fp(_F_SEMI, 8.5), color=_MUTED) fig.text(0.015, 0.905, "ARPAbet \u2192 viseme \u2192 mouth, on the phoneme timeline", fontproperties=_fp(_F_DISPLAY, 16), color=_INK) fig.text(0.5, 0.005, f"top: the ten shapes the renderer makes \u00b7 " f"bottom: the mouth saying \u201c{sentence}\u201d in real time", ha="center", fontproperties=_fp(_F, 9), color=_MUTED) fig.savefig(path, bbox_inches="tight", pad_inches=0.22) plt.close(fig) return path def _fig_simultaneity(path): """Why the channels agreeing by construction is the whole point.""" plt = _plt() fig, axes = plt.subplots(1, 2, figsize=(11.2, 3.6), dpi=170) rows = ["what it says", "what you hear", "what the face does"] panels = [("three models, three clocks", [0.0, 0.24, 0.44], "every gap is latency, and every gap is where the\nchannels can disagree", _WARM), ("one trunk, one clock", [0.0, 0.0, 0.0], "one decision drives the word, the sound and the\nmouth at the same instant", _COOL)] for ax, (title, offsets, note, col) in zip(axes, panels): _blank(ax) ax.text(0.5, 0.96, title, ha="center", va="top", fontproperties=_fp(_F_SEMI, 13), color=col) for i, (r, off) in enumerate(zip(rows, offsets)): y = 0.68 - i * 0.20 ax.text(0.0, y, r, fontproperties=_fp(_F, 10), color=_BODY, va="center") for k in range(6): x = 0.42 + k * 0.105 + off if x > 0.99: continue ax.plot([x, x], [y - 0.055, y + 0.055], color=col, lw=2.6 if not off else 2.0, solid_capstyle="round") ax.text(0.5, 0.02, note, ha="center", va="bottom", fontproperties=_fp(_F, 9), color=_MUTED, linespacing=1.5) axes[0].annotate("", xy=(0.42, 0.885), xytext=(0.48, 0.885), arrowprops=dict(arrowstyle="<->", color=_WARM, lw=1.1)) axes[0].text(0.45, 0.925, "offset", fontproperties=_fp(_F, 8), color=_WARM, ha="center") fig.text(0.5, 1.06, "EMOTION IS A PROPERTY OF SIMULTANEITY, NOT A FOURTH MODEL", ha="center", fontproperties=_fp(_F_SEMI, 9), color=_MUTED) fig.tight_layout() fig.savefig(path, bbox_inches="tight", pad_inches=0.24) plt.close(fig) return path def _fig_metrics(path, runs=None, tail=40000): """Real training curves, straight out of runs/*/log.jsonl.""" plt = _plt() want = runs or [("vlm", "loss", "text / VQA loss", _ACCENT), ("captions", "loss", "captions (real photos)", _COOL), ("ear", "ctc_loss", "ear CTC", _GOLD), ("vocoder", "stft_loss", "vocoder STFT", _WARM)] fig, axes = plt.subplots(2, 2, figsize=(11.2, 5.6), dpi=170) for ax, (stage, key, title, col) in zip(axes.ravel(), want): drew = False for d in sorted(Path(RUN_ROOT).glob(f"{stage}-*")): log = d / "log.jsonl" if not log.exists(): continue steps, vals = [], [] for line in log.read_text(errors="replace").splitlines()[-tail:]: try: r = json.loads(line) except Exception: continue if isinstance(r.get(key), (int, float)): steps.append(r.get("step") or 0) vals.append(r[key]) if len(steps) > 2: tag = "-".join(d.name.split("-")[2:3]) or d.name ax.plot(steps, vals, lw=1.6, color=col, alpha=0.9, label=f"{tag} @{int(max(steps)):,}") drew = True ax.set_title(title, fontproperties=_fp(_F_SEMI, 11), color=_INK, pad=8, loc="left") ax.grid(axis="y", alpha=0.9) ax.set_axisbelow(True) ax.tick_params(length=0) for s in ax.spines.values(): s.set_color(_RULE) if drew: ax.legend(loc="upper right") else: ax.text(0.5, 0.5, "not started", ha="center", va="center", fontproperties=_fp(_F, 9), color=_MUTED, transform=ax.transAxes) fig.text(0.02, 0.985, "MEASURED ON THIS MACHINE \u00b7 RTX PRO 6000 BLACKWELL", fontproperties=_fp(_F_SEMI, 8.5), color=_MUTED) fig.tight_layout(rect=(0.0, 0.0, 1.0, 0.955)) fig.savefig(path, bbox_inches="tight", pad_inches=0.22) plt.close(fig) return path def _fig_roadmap(path): """The whole project as one picture: what each stage is, how far it has run, and how many checkpoints it has produced. Read live from `runs/*/meta/state.json` and the same launch plan the swarm uses, so the figure cannot drift from the training it describes. """ plt = _plt() #: stage -> (plan_steps, ckpt_every, what it produces, kind) #: kind: "pre" = trained from random weights, "post" = built on/beside the trunk PLAN = [("vlm", 8000, 500, "eyes + trunk, from random weights", "pre"), ("captions", 500000, 500, "real photographs + captions", "post"), ("ear", 400000, 3000, "audio -> phonemes (CTC)", "post"), ("speech", 150000, 500, "voice + durations + mel", "post"), ("vocoder", 700000, 6000, "mel -> waveform", "post"), ("align", 0, 0, "forced alignment for speech", "done")] rows = [] for stage, plan, ck, what, kind in PLAN: done = 0 for d in sorted(Path(RUN_ROOT).glob(f"{stage}-*")): f = d / "meta" / "state.json" if f.exists(): done = max(done, int(json.loads(f.read_text()).get("step") or 0)) frac = (done / plan) if plan else 1.0 rows.append((stage, plan, done, min(frac, 1.0), ck, what, kind)) fig, ax = plt.subplots(figsize=(11.2, 4.9), dpi=170) _blank(ax) _eyebrow(ax, "roadmap · status · checkpoints", 0.012, 0.985) _h1(ax, "Five stages, one trunk, and where each one stands", 0.012, 0.905) y = 0.735 bw, x0 = 0.455, 0.300 for stage, plan, done, frac, ck, what, kind in rows: col = _ACCENT if kind == "pre" else (_COOL if kind == "done" else _INK) ax.text(0.012, y, stage, fontproperties=_fp(_F_SEMI, 12.5), color=col, va="center") ax.text(0.012, y - 0.062, what, fontproperties=_fp(_F, 9), color=_MUTED, va="center") # the track ax.add_patch(plt.Rectangle((x0, y - 0.026), bw, 0.052, facecolor="#EDEFF4", edgecolor="none", zorder=1)) ax.add_patch(plt.Rectangle((x0, y - 0.026), max(bw * frac, 0.004), 0.052, facecolor=col, edgecolor="none", zorder=2)) if kind == "done": label = "complete" chips = [(0.0, "done", _COOL)] else: label = f"{done:,} / {plan:,} steps" chips = [(frac * 100, f"{frac*100:.0f}%", col)] ax.text(x0 + bw + 0.018, y + 0.004, label, fontproperties=_fp(_F_MED, 9.5), color=_BODY, va="center") ck_n = (done // ck) if ck else 0 ax.text(x0 + bw + 0.018, y - 0.054, f"{ck_n:,} checkpoints written" + (f" · every {ck:,} steps" if ck else ""), fontproperties=_fp(_F, 8.5), color=_MUTED, va="center") y -= 0.118 fig.text(0.012, 0.045, "checkpoint every N steps · `latest/` is overwritten each time, so only " "milestones accumulate as distinct records", fontproperties=_fp(_F, 8.5), color=_MUTED) fig.savefig(path, bbox_inches="tight", pad_inches=0.24) plt.close(fig) return path def build_card_figures(log=print): """Every card figure. One failure must not cost the others, so each is tried.""" CARD_ASSETS.mkdir(parents=True, exist_ok=True) made = {} for name, fn in (("hero", _fig_hero), ("roadmap", _fig_roadmap), ("pipeline", _fig_pipeline), ("architecture", _fig_architecture), ("scenes", _fig_scenes), ("visemes", _fig_visemes), ("simultaneity", _fig_simultaneity), ("metrics", _fig_metrics)): out = CARD_ASSETS / f"{name}.png" try: fn(out) made[name] = out log(f" {name:14s} {out.stat().st_size/1024:7.1f} KB") except Exception as exc: log(f" {name:14s} FAILED: {type(exc).__name__} {exc}") return made card_figures = build_card_figures() print(f"{len(card_figures)} figures -> {CARD_ASSETS}") return @app.cell(hide_code=True) def publish(CFG, HF_TOKEN, OUT_DIR, hub, hub_status, infer, model_card): # --- Publish: notebook + training code + weights + model card --------------- def train_stats(): """Latest known state per stage; the Hub is the source of truth.""" stats = {} for row in hub_status(): stats[row["stage"]] = row return stats def bundle_weights(path="/marimo/artifacts/omni_weights.pt", log=print): """One file with every trained part, from the checkpoints each stage serves. Two things this has to get right, and previously did not: * it referenced an undefined name `rid`, so it raised `NameError` on the first stage it bundled -- it had never once produced a file; * it carried no vocabulary. The trunk's embedding and the word list are two halves of one token space, and a bundle holding a v2 trunk (16 380 words) beside a v1 word list (8 011) loads perfectly and then answers fluent nonsense. That is not a hypothetical: it is what shipped in `weights/` and what a stranger's `make_movie.py` would have reproduced. The list now travels inside the bundle, chosen to fit the trunk's own embedding. """ import torch out = {"meta": {"preset": CFG["preset"], "img_size": CFG["img_size"], "data_seed": CFG["data_seed"], "stages": {}}} trunk, rows = None, None for stage in ("speech", "vlm", "ear", "vocoder"): store, _state = hub.open_stage_run(CFG["hf_repo"], stage, HF_TOKEN) ck = store.load_latest(tag="model") if store else None if not (store and ck and ck.get("payload")): log(f" {stage}: nothing to bundle") continue out[stage] = ck["payload"] out["meta"]["stages"][stage] = {"run_id": store.run_id, "step": ck["step"], "from_remote": bool(ck["from_remote"])} log(f" bundled {stage}: {store.run_id} step {ck['step']}") if trunk is None and stage in ("speech", "vlm") and "tok.weight" in ck["payload"]: trunk = ck["payload"] rows = int(trunk["tok.weight"].shape[0]) words = None if rows: v = infer.vocab_for_rows(rows, log=log) if v is not None: words = v.words out["meta"]["n_words"], out["meta"]["n_phones"] = v.n_words, v.n_phones if words is None: words = json.loads(Path("/marimo/runs/shared/vocab_words.json").read_text()) log(f" WARNING: no vocabulary matched the trunk, bundling v1 ({len(words)} words); " f"decoding may be wrong for whoever downloads this") out["meta"]["vocab_words"] = words out["meta"]["vocab_size"] = len(words) out["meta"]["model"] = infer.default_cfg(preset=CFG["preset"], data_seed=CFG["data_seed"], img_size=CFG["img_size"])["model"] Path(path).parent.mkdir(parents=True, exist_ok=True) torch.save(out, path) return {"path": path, "gb": round(Path(path).stat().st_size / 1e9, 2), "parts": list(out["meta"]["stages"]), "words": len(words)} def publish(repo=None, message="VLM-all-inclusive", with_weights=True, with_media=True, log=print): """Push code + notebook + model card (+ weights and sample movies).""" from huggingface_hub import create_repo repo = repo or CFG["public_repo"] token = HF_TOKEN or None create_repo(repo, repo_type="model", private=False, exist_ok=True, token=token) stats = train_stats() demo = Path(OUT_DIR) / "demo_report.json" extra = "" if demo.exists(): rows = json.loads(demo.read_text()) extra = ("| case | question | answer | ground truth | exact |\n|---|---|---|---|---|\n" + "\n".join(f"| {r['seed']} | {r['question']} | {r['answer']} | {r['truth']} " f"| {'yes' if r.get('exact') else 'no'} |" for r in rows) + f"\n\n_Measured on this machine: mean word overlap " f"{sum(r.get('word_overlap', 0) for r in rows)/max(1, len(rows)):.2f}, " f"audio RMS {sum(r.get('wav_rms', 0) for r in rows)/max(1, len(rows)):.3f}, " f"mouth aperture range " f"{min(r['mouth_range'][0] for r in rows):.2f}-" f"{max(r['mouth_range'][1] for r in rows):.2f}._") Path("/marimo/README.md").write_text(model_card(stats, extra=extra)) log(f"publishing to https://huggingface.co/{repo}") # ONE commit for the whole publication: code, notebook, card, samples, weights items = [("/marimo/README.md", "README.md"), ("/marimo/notebook.py", "notebook.py")] items += [(str(p), f"omni/{p.name}") for p in sorted(Path("/marimo/omni").glob("*.py"))] if with_media: items += [(str(p), f"samples/{p.name}") for p in sorted(Path(OUT_DIR).glob("*.mp4")) + sorted(Path(OUT_DIR).glob("*.wav")) + sorted(Path(OUT_DIR).glob("*.png"))] if with_weights: info = bundle_weights(log=log) items.append((info["path"], "weights/omni_weights.pt")) log(f" weights bundle {info['gb']} GB ({', '.join(info['parts'])})") writer = hub.RemoteRepo(repo, token=token, private=False) res = writer.commit(items, message or "VLM-all-inclusive") log(f"committed {len(items)} files in one commit" + ("" if res else " (FAILED - see the log, likely the API rate limit)")) return repo return (publish,) @app.cell(hide_code=True) def launch_controls(CFG, STAGE_HELP): # --- Training controls (widgets only: marimo forbids reading a widget's value in # --- the same cell that creates it) ------------------------------------------- stage_picker = mo.ui.dropdown(list(STAGE_HELP), value="vlm", label="stage") steps_input = mo.ui.number(value=CFG["steps"], start=10, stop=500000, step=10, label="steps") batch_input = mo.ui.number(value=CFG["batch"], start=1, stop=4096, step=1, label="batch") extra_input = mo.ui.text(value="", full_width=True, label="extra CLI args") start_btn = mo.ui.run_button(label="start / resume") resync_btn = mo.ui.run_button(label="resync local -> Hub") launch_controls = mo.vstack([ mo.md("## Train (detached, resumable)"), mo.hstack([stage_picker, steps_input, batch_input], justify="start", gap=1), extra_input, mo.md("\n".join(f"- `{k}` - {v}" for k, v in STAGE_HELP.items())), mo.hstack([start_btn, resync_btn], justify="start"), ]) return ( batch_input, extra_input, resync_btn, stage_picker, start_btn, steps_input, ) @app.cell(hide_code=True) def launch_panel( CFG, HF_TOKEN, batch_input, extra_input, hub, launch, resync_btn, running, stage_picker, start_btn, steps_input, ): # --- Training status / launch action ---------------------------------------- _cmd = launch(stage_picker.value, steps=steps_input.value, batch=batch_input.value, extra=extra_input.value) if start_btn.value else None _resynced = [] if resync_btn.value: _resynced = hub.resync_local_runs(CFG["hf_repo"], token=HF_TOKEN, retries=3) launch_panel = mo.vstack([ mo.md("resynced: " + ", ".join(_resynced)) if _resynced else mo.md(""), mo.md(f"```\n{_cmd}\n```") if _cmd else mo.md("_idle - press **start / resume**; pressing it again resumes from the last checkpoint_"), mo.ui.table(running() or [{"pid": "-", "stage": "nothing running"}], selection=None, label="local processes"), ]) return @app.cell(hide_code=True) def prompt_input(CFG, HF_TOKEN, infer, scenes): # ============================================================================ # CELL 1 of 2 - the prompt # ============================================================================ # Type what you want the model to answer about the picture, press generate, then # play the result in cell 2 (the mp4 carries its own audio track, so the video # and the voice stay in sync). SYSTEMS = {} # loaded weights, cached in the kernel def get_system(log=print): """Local run dirs if this box has them, else the published weights bundle.""" key = (CFG["preset"], CFG["run_tag"], CFG["data_seed"], CFG["img_size"]) if key not in SYSTEMS: cfg = infer.default_cfg(preset=CFG["preset"], data_seed=CFG["data_seed"], img_size=CFG["img_size"], run_tag=CFG["run_tag"], hf_repo=CFG["hf_repo"]) system = infer.OmniSystem.load(cfg, token=HF_TOKEN, log=log) if system.model is None: # fresh box: no checkpoints on disk log("no local checkpoints - pulling the published bundle") system = infer.OmniSystem.from_bundle(CFG["public_repo"], token=HF_TOKEN, log=log) SYSTEMS[key] = system return SYSTEMS[key] def fresh_scene(seed=None): """A new test image plus the question/answer the generator knows are true.""" rng = np.random.default_rng(int(seed if seed is not None else time.time())) scene = scenes.make_scene(rng) q, a = scenes.question_answer(scene, rng) return scene, q, a prompt_box = mo.ui.text_area(value="describe the image", label="prompt", full_width=True, rows=2) prompt_image = mo.ui.file(filetypes=[".png", ".jpg", ".jpeg", ".webp"], label="image (optional - otherwise a fresh scene is rendered)") prompt_seed = mo.ui.number(value=int(time.time()) % 99991, start=0, stop=10 ** 6, label="scene seed") prompt_size = mo.ui.dropdown({"384 px": 384, "512 px": 512}, value="384 px", label="video size") generate_btn = mo.ui.run_button(label="generate video with sound") reload_btn = mo.ui.run_button(label="reload weights") prompt_input = mo.vstack([ mo.md("## 1. Prompt"), prompt_box, mo.hstack([prompt_image, mo.vstack([prompt_seed, prompt_size]), generate_btn, reload_btn], gap=1, align="start"), mo.md("_The answer is generated by the model, spoken by its own voice head and vocoder, " "and lip-synced by its own phoneme timeline. The mp4 has the audio inside it._"), ]) return ( SYSTEMS, fresh_scene, generate_btn, get_system, prompt_box, prompt_image, prompt_seed, prompt_size, reload_btn, ) @app.cell(hide_code=True) def publish_controls(CFG): # --- Publish controls (widgets only) ----------------------------------------- publish_btn = mo.ui.run_button(label="publish to the public repo") publish_controls = mo.vstack([mo.md("## Publish"), publish_btn, mo.md(f"target: `{CFG['public_repo']}`")]) return (publish_btn,) @app.cell(hide_code=True) def publish_panel(publish, publish_btn): # --- Publish action ---------------------------------------------------------- preview = mo.md("_nothing published yet_") if publish_btn.value: try: preview = mo.md(f"published -> https://huggingface.co/{publish()}") except Exception: import traceback as _tb preview = mo.callout(mo.md(f"```\n{_tb.format_exc()[-1500:]}\n```"), kind="danger") publish_panel_preview = preview return @app.cell(hide_code=True) def eval_controls(): # --- Eval controls ----------------------------------------------------------- eval_n = mo.ui.slider(start=20, stop=400, value=120, step=20, label="scenes") eval_btn = mo.ui.run_button(label="run eval") eval_controls = mo.vstack([mo.md("## Eval"), mo.hstack([eval_n, eval_btn])]) return eval_btn, eval_n @app.cell(hide_code=True) def eval_panel(eval_btn, eval_n, get_system, pd, scenes): # --- Eval: exact-match accuracy per question type (free generation) ---------- if eval_btn.value: _sys = get_system() rows = [] rng = np.random.default_rng(2024) _t0 = time.time() for _i in range(int(eval_n.value)): _scene = scenes.make_scene(rng) _q, _truth = scenes.question_answer(_scene, rng) _img = scenes.render_scene(_scene, px=252) _pred = _sys.answer(_img, _q, max_new=24) rows.append({"q": _q, "pred": _pred, "truth": _truth, "exact": _pred.strip() == _truth.strip()}) _df = pd.DataFrame(rows) _df["kind"] = _df["q"].str.split().str[0] _summary = (_df.groupby("kind").agg(n=("exact", "size"), exact=("exact", "mean")) .sort_values("n", ascending=False).reset_index()) _summary["exact"] = (_summary["exact"] * 100).round(1) EVAL_RESULT = {"exact": round(float(_df["exact"].mean()) * 100, 1), "n": len(_df), "seconds": round(time.time() - _t0, 1), "by_kind": _summary.to_dict("records"), "misses": _df[~_df["exact"]].head(8).to_dict("records")} print(f"exact {EVAL_RESULT['exact']}% over {EVAL_RESULT['n']} questions " f"({EVAL_RESULT['seconds']}s)", flush=True) else: EVAL_RESULT = None if EVAL_RESULT: eval_panel = mo.vstack([ mo.md(f"**exact match {EVAL_RESULT['exact']}%** over {EVAL_RESULT['n']} freshly " f"generated questions ({EVAL_RESULT['seconds']}s)"), mo.ui.table(EVAL_RESULT["by_kind"], selection=None), mo.accordion({"examples it got wrong": mo.ui.table(EVAL_RESULT["misses"], selection=None)}), ]) else: eval_panel = mo.md("_press run eval: it generates fresh scenes, asks the VLM, and scores " "exact matches per question type_") return @app.cell(hide_code=True) def lib_config(): # ===== omni/config.py ===== single source of truth for run identity # Imports come from `setup`; `materialize` writes this back out to /marimo/omni/config.py STAGE_NAMES = ("vlm", "ear", "align", "speech", "vocoder", "captions") class ConfigError(ValueError): """Raised for an invalid or inconsistent configuration.""" #: Model geometry per preset. Parameter counts are measured, not estimated. PRESETS: dict[str, dict[str, int]] = { "nano": dict(d_model=256, n_layers=4, n_heads=4, vision_d_model=192, vision_layers=3, vision_heads=3), "small": dict(d_model=384, n_layers=6, n_heads=6, vision_d_model=256, vision_layers=4, vision_heads=4), "base": dict(d_model=512, n_layers=8, n_heads=8, vision_d_model=384, vision_layers=6, vision_heads=6), "large": dict(d_model=768, n_layers=12, n_heads=12, vision_d_model=512, vision_layers=8, vision_heads=8), "xl": dict(d_model=1024, n_layers=16, n_heads=16, vision_d_model=640, vision_layers=10, vision_heads=10), "xxl": dict(d_model=1280, n_layers=24, n_heads=16, vision_d_model=768, vision_layers=12, vision_heads=12), } #: Keys that make a run a *different* run (architecture + data + token space). IDENTITY_KEYS = ("preset", "model", "img_size", "data_seed", "data_version", "vocab_size", "vocab_hash") def preset_names() -> tuple[str, ...]: return tuple(PRESETS) @dataclass class RunConfig: """Everything a stage needs, with defaults that work on a 100 GB GPU.""" # --- identity ----------------------------------------------------------- preset: str = "xl" img_size: int = 128 data_seed: int = 0 data_version: int = 2 # omni.scenes.DATA_VERSION; see that module run_tag: str = "main" model: dict[str, Any] = field(default_factory=dict) # filled from `preset` vocab_size: int = 0 # filled by the vocab builder vocab_hash: str = "" # --- optimisation ------------------------------------------------------- steps: int = 8000 batch: int = 384 accum: int = 1 lr: float = 4e-4 wd: float = 0.05 warmup: int = 200 clip: float = 1.0 amp_dtype: str = "bf16" channels_last: bool = True log_every: int = 20 lookahead: int = 6 minutes: float = 0.0 # wall-clock budget for a stage (0 = none) # --- checkpointing / durability ---------------------------------------- ckpt_steps: int = 250 # local checkpoint cadence ckpt_seconds: float = 900.0 push_seconds: float = 1200.0 # Hub commit cadence (rate limited, so slower) keep_milestones: int = 3 milestone_every: int = 5 resume: str = "auto" # auto | never # --- repositories ------------------------------------------------------- hf_repo: str = "hkblockchaintechnology/omni-vlm-ckpt" # public: checkpoints (weights, optimizer, metrics, logs) public_repo: str = "hkblockchaintechnology/VLM-all-inclusive" # code + card # --- stage-specific ----------------------------------------------------- workers: int = 12 # CPU renderer threads (legacy path) ear_lr: float | None = None sp_lr: float | None = None voc_lr: float | None = None vl_mix: float = 0.25 # fraction of vision-language batches during TTS sp_max_sec: float = 8.0 sp_seed: int = 1 ear_max_sec: float = 6.0 voc_crop_sec: float = 1.0 align_limit: int = 0 # 0 = whole corpus svc: bool = False # reserved for service-mode flags # --- vocabulary + warm start -------------------------------------------- #: "v1" = synthetic scenes + speech; "v2" = those plus real photo captions. vocab: str = "v1" #: stage whose newest run seeds the trunk ("" = train from scratch). This is #: how the voice heads join the real-photo trunk instead of living on a #: separate model that inference would never serve. base_stage: str = "" # --- vocoder GAN (fixes the flat-audio failure mode of a pure STFT loss) --- adversarial: bool = False disc_lr: float = 2e-4 adv_weight: float = 1.0 fm_weight: float = 2.0 # --- real-data caption stage --------------------------------------------- scene_mix: float = 0.3 # fraction of synthetic Q/A batches mixed in max_len: int = 32 # caption length cap (word tokens) def __post_init__(self) -> None: if not self.model: self.model = dict(PRESETS[self.preset]) if self.preset in PRESETS else {} self.model.setdefault("img_size", self.img_size) self.validate() # ------------------------------------------------------------------ checks def validate(self) -> "RunConfig": """Fail loudly and specifically instead of somewhere deep in a step.""" if self.preset not in PRESETS: raise ConfigError(f"unknown preset {self.preset!r}; choose from {sorted(PRESETS)}") for name, value, lo in (("steps", self.steps, 1), ("batch", self.batch, 1), ("accum", self.accum, 1), ("log_every", self.log_every, 1), ("ckpt_steps", self.ckpt_steps, 1), ("img_size", self.img_size, 16)): if int(value) < lo: raise ConfigError(f"{name} must be >= {lo}, got {value!r}") if self.lr <= 0: raise ConfigError(f"lr must be positive, got {self.lr!r}") if self.amp_dtype not in ("bf16", "fp16", "no"): raise ConfigError("amp_dtype must be bf16, fp16 or no") if self.resume not in ("auto", "never"): raise ConfigError("resume must be auto or never") if self.vocab not in ("v1", "v2"): raise ConfigError("vocab must be v1 or v2") if not 0.0 <= self.vl_mix <= 1.0: raise ConfigError("vl_mix is a fraction in [0, 1]") if self.img_size % self.model.get("patch", 8) != 0: raise ConfigError(f"img_size {self.img_size} must be divisible by patch " f"{self.model.get('patch', 8)}") if self.model.get("d_model", 0) % self.model.get("n_heads", 1) != 0: raise ConfigError("d_model must be divisible by n_heads") return self # ------------------------------------------------------------ conversions def to_dict(self) -> dict[str, Any]: return asdict(self) @classmethod def from_dict(cls, data: Mapping[str, Any]) -> "RunConfig": known = {f.name for f in fields(cls)} unknown = set(data) - known if unknown: raise ConfigError(f"unknown configuration keys: {sorted(unknown)}") return cls(**dict(data)) def with_overrides(self, **overrides: Any) -> "RunConfig": """A validated copy. `None` values are ignored so CLIs can pass defaults.""" clean = {k: v for k, v in overrides.items() if v is not None} if "model" in clean and clean["model"] is None: clean.pop("model") merged = replace(self, **clean) merged.validate() return merged def to_json(self) -> str: return json.dumps(self.to_dict(), indent=2, sort_keys=True, default=str) # -------------------------------------------------------------- identity def identity(self) -> dict[str, Any]: """The projection that decides whether a resume is legitimate.""" return {k: getattr(self, k) for k in IDENTITY_KEYS if hasattr(self, k)} def fingerprint(self, stage: str | None = None) -> str: payload = {**self.identity(), "stage": stage} if stage else self.identity() blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) return hashlib.sha256(blob.encode()).hexdigest() def run_id(self, stage: str) -> str: """`---`: stable across processes.""" if stage not in STAGE_NAMES: raise ConfigError(f"unknown stage {stage!r}; choose from {list(STAGE_NAMES)}") return f"{stage}-{self.preset}-{self.run_tag or 'main'}-{self.fingerprint(stage)[:6]}" # ------------------------------------------------------------ convenience def stage_lr(self, stage: str) -> float: return {"ear": self.ear_lr, "speech": self.sp_lr, "vocoder": self.voc_lr}.get( stage) or self.lr return PRESETS, RunConfig, STAGE_NAMES @app.cell(hide_code=True) def lib_logging_utils(): # ===== omni/logging_utils.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/logging_utils.py using only the imports this module actually needs. # --------------------------------------------------------------------------- # omni/logging_utils.py - one logging setup for every process. # # Training runs detached for hours, so logs have two audiences: a human tailing # a file, and the notebook parsing metrics. Human lines go to stderr, metrics # go to a JSONL file, and both carry the run/stage context. # --------------------------------------------------------------------------- _CONFIGURED = False _FMT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s" class _HumanFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: base = super().format(record) extra = getattr(record, "context", None) if extra: base += " " + " ".join(f"{k}={v}" for k, v in extra.items()) return base def configure_logging(level: str | int | None = None, stream=None) -> None: """Idempotent process-wide setup (safe to call from every entry point).""" global _CONFIGURED if _CONFIGURED: return lvl = level if level is not None else os.environ.get("OMNI_LOG_LEVEL", "INFO") handler = logging.StreamHandler(stream or sys.stderr) handler.setFormatter(_HumanFormatter(_FMT, datefmt="%H:%M:%S")) root = logging.getLogger("omni") root.handlers[:] = [handler] root.setLevel(lvl) root.propagate = False _CONFIGURED = True def get_logger(name: str) -> logging.Logger: configure_logging() return logging.getLogger(f"omni.{name}") class JsonlMetrics: """Append-only metric log: one JSON object per line, flushed immediately. Flushing matters: the sandbox can be reaped mid-run, and an unflushed metric is a metric that never existed. A torn final line is tolerated by the reader (it skips unparseable lines). """ def __init__(self, path: str | Path, **base: Any) -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) self.base = base self._logger = get_logger("metrics") def write(self, **metrics: Any) -> dict[str, Any]: record = {"t": time.time(), "wall": time.strftime("%H:%M:%S"), **self.base, **metrics} with open(self.path, "a", encoding="utf-8") as fh: fh.write(json.dumps(record, default=str) + "\n") fh.flush() os.fsync(fh.fileno()) return record def read(self, limit: int | None = None) -> list[dict[str, Any]]: if not self.path.exists(): return [] rows: list[dict[str, Any]] = [] for line in self.path.read_text(encoding="utf-8", errors="replace").splitlines(): try: rows.append(json.loads(line)) except json.JSONDecodeError: # torn write from a killed process continue return rows[-limit:] if limit else rows def tail(self, n: int = 20) -> list[dict[str, Any]]: return self.read(limit=n) return configure_logging, get_logger @app.cell(hide_code=True) def lib_doctor(): # ===== omni/doctor.py ===== module body # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/doctor.py using only the imports this module actually needs. # --------------------------------------------------------------------------- # omni/doctor.py - environment and repository diagnostics. # # Every failure mode this project actually hit is a check here: a sandbox # without ffmpeg, a missing token, a Hub commit that silently fails, a stale # local checkpoint that the Hub never received, a GPU too full to start. # # python -m omni.doctor # human table, exit 1 if a required check fails # python -m omni.doctor --json # machine readable # --------------------------------------------------------------------------- REQUIRED_PACKAGES = ("numpy", "PIL", "torch", "huggingface_hub") OPTIONAL_PACKAGES = ("scipy", "pandas", "altair", "marimo", "soundfile") @dataclass class Check: name: str ok: bool detail: str = "" fix: str = "" required: bool = True def __str__(self) -> str: # pragma: no cover - presentation only mark = "PASS" if self.ok else ("FAIL" if self.required else "warn") line = f"[{mark}] {self.name:<26} {self.detail}" return line + (f"\n -> {self.fix}" if (not self.ok and self.fix) else "") def _check(name: str, fn, fix: str = "", required: bool = True) -> Check: try: detail = fn() or "" return Check(name, True, str(detail), "", required) except Exception as exc: # noqa: BLE001 - diagnostics return Check(name, False, f"{type(exc).__name__}: {exc}"[:160], fix, required) def run_checks(repo_id: str | None = None, token: str | None = None, root: str = "/marimo", probe_commit: bool = True) -> list[Check]: """Run every diagnostic. Never raises: a failing check is data.""" checks: list[Check] = [] checks.append(_check("python", lambda: f"{sys.version.split()[0]}", "use python >= 3.10")) checks.append(_check("packages.required", lambda: ", ".join(f"{p}" for p in REQUIRED_PACKAGES if importlib.util.find_spec(p) is None) or "all present", "pip install -e .")) optional = [p for p in OPTIONAL_PACKAGES if importlib.util.find_spec(p) is None] checks.append(_check("packages.optional", lambda: ", ".join(optional) or "all present", "optional - some features degrade", required=False)) def _torch(): import torch line = f"torch {torch.__version__}" if torch.cuda.is_available(): line += f" | cuda {torch.version.cuda} | {torch.cuda.get_device_name(0)}" return line checks.append(_check("torch", _torch, "install a CUDA build of torch")) def _gpu(): import torch if not torch.cuda.is_available(): raise RuntimeError("no CUDA device") free, total = torch.cuda.mem_get_info() return (f"{torch.cuda.device_count()} device(s), {free/2**30:.1f} GiB free of " f"{total/2**30:.1f} GiB") checks.append(_check("gpu.memory", _gpu, "free GPU memory or lower --batch")) def _ffmpeg(): exe = shutil.which("ffmpeg") if not exe: raise FileNotFoundError("ffmpeg not on PATH") return exe checks.append(_check("ffmpeg", _ffmpeg, "install ffmpeg (needed to mux the movies)")) def _dirs(): made = [] for sub in ("runs", "logs", "out", "omni", "artifacts"): p = Path(root) / sub p.mkdir(parents=True, exist_ok=True) probe = p / ".doctor_probe" probe.write_text("ok") probe.unlink() made.append(sub) return "writable: " + ", ".join(made) checks.append(_check("workspace", _dirs, f"make {root} writable")) def _token(): tok = token or os.environ.get("HF_TOKEN") or ( (Path(root) / ".hf_token").read_text().strip() if (Path(root) / ".hf_token").exists() else "") if not tok: raise FileNotFoundError("no HF token") return f"present ({tok[:7]}...)" checks.append(_check("hf.token", _token, "export HF_TOKEN or write /marimo/.hf_token")) def _whoami(): import huggingface_hub as hf tok = token or os.environ.get("HF_TOKEN") or ( (Path(root) / ".hf_token").read_text().strip() if (Path(root) / ".hf_token").exists() else "") who = hf.whoami(token=tok) return f"authenticated as {who.get('name')}" checks.append(_check("hf.auth", _whoami, "check the token scope (repo.write)")) if repo_id: def _repo_read(): import huggingface_hub as hf tok = token or os.environ.get("HF_TOKEN") or ( (Path(root) / ".hf_token").read_text().strip() if (Path(root) / ".hf_token").exists() else "") files = hf.HfApi(token=tok).list_repo_files(repo_id) return f"{repo_id}: {len(files)} files" checks.append(_check("hf.repo.read", _repo_read, "create the repo or check the name")) def _repo_commit(): from huggingface_hub import CommitOperationAdd, create_commit tok = token or os.environ.get("HF_TOKEN") probe = Path(root) / "runs" / ".doctor_commit" probe.parent.mkdir(parents=True, exist_ok=True) probe.write_text(str(time.time())) t0 = time.time() create_commit(repo_id=repo_id, operations=[CommitOperationAdd(path_in_repo=".doctor/probe.txt", path_or_fileobj=str(probe))], token=tok, commit_message="doctor probe") return f"commit round trip in {time.time()-t0:.1f}s" checks.append(_check("hf.repo.commit", _repo_commit if probe_commit else lambda: "skipped", "commits are rate limited; batch them (hub.RemoteRepo.commit)", required=probe_commit)) def _corpus(): meta = Path("/marimo/data/LJSpeech-1.1/metadata.csv") if not meta.exists(): raise FileNotFoundError("LJSpeech metadata.csv missing") wavs = len(list((meta.parent / "wavs").glob("*.wav"))) return f"{wavs} clips, {meta.parent}" checks.append(_check("data.ljspeech", _corpus, "download LJSpeech-1.1 into /marimo/data", required=False)) def _cmudict(): p = Path("/marimo/data/cmudict.dict") if not p.exists(): raise FileNotFoundError("cmudict.dict missing") return f"{len(p.read_text(errors='replace').splitlines())} entries" checks.append(_check("data.cmudict", _cmudict, "download cmudict.dict", required=False)) def _vocab(): p = Path(root) / "runs" / "shared" / "vocab_words.json" if not p.exists(): raise FileNotFoundError("vocab cache missing (built on first run)") return f"{len(json.loads(p.read_text()))} words" checks.append(_check("data.vocab", _vocab, "any train stage will build it", required=False)) return checks def doctor_main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser("omni.doctor", description="environment diagnostics") ap.add_argument("--repo", default=None, help="check read/write access to this HF repo") ap.add_argument("--root", default="/marimo") ap.add_argument("--json", action="store_true") ap.add_argument("--no-commit-probe", action="store_true") args = ap.parse_args(argv) checks = run_checks(repo_id=args.repo, root=args.root, probe_commit=not args.no_commit_probe) if args.json: print(json.dumps([asdict(c) for c in checks], indent=2)) else: for c in checks: print(c) failed = [c for c in checks if not c.ok and c.required] if not args.json: print(f"\n{len(checks) - len(failed)}/{len(checks)} checks passed" + (f" | {len(failed)} required failure(s)" if failed else "")) return 1 if failed else 0 return @app.cell(hide_code=True) def lib_cli( PRESETS: dict[str, dict[str, int]], RunConfig, STAGE_NAMES, configure_logging, get_logger, ): # ===== omni/cli.py ===== module body (imports come from `setup`) log = get_logger("cli") def _cfg(args) -> RunConfig: base = RunConfig.from_dict(json.loads(args.config)) if args.config else RunConfig() overrides = {} for name in ("preset", "steps", "batch", "accum", "lr", "img_size", "data_seed", "log_every", "minutes", "run_tag", "resume", "ckpt_steps", "push_seconds", "hf_repo", "public_repo", "vl_mix"): value = getattr(args, name, None) if value is not None: overrides[name] = value for item in args.set or []: key, _, raw = item.partition("=") try: overrides[key] = json.loads(raw) except json.JSONDecodeError: overrides[key] = raw cfg = base.with_overrides(**overrides) # A typed config cannot know the vocabulary until it is built, so it is filled # in here -- and it is part of the run identity. The real-data stage has its # own, larger vocabulary (real captions + speech + scenes); stamping the # vision-language one on it is how its run record came to disagree with the # embedding table it actually trained. import omni.data as od if getattr(args, "stage", "") == "captions" or cfg.vocab == "v2": import omni.captions as _caps full = _caps.vocab_identity_v2() ident = {k: v for k, v in full.items() if k in ("vocab_size", "vocab_hash")} else: ident = od.vocab_identity(cfg.data_seed) return cfg.with_overrides(**ident) def cmd_doctor(args) -> int: from omni.doctor import doctor_main return doctor_main([*(("--repo", args.repo) if args.repo else ()), *(("--json",) if args.json else ())]) def cmd_stages(args) -> int: cfg = _cfg(args) import omni.hub as hub token = _token() rows = [] for stage in STAGE_NAMES: store, state = hub.open_stage_run(cfg.hf_repo, stage, token) local = Path("/marimo/runs") / cfg.run_id(stage) / "meta" / "state.json" rows.append({ "stage": stage, "run_id": store.run_id if store else cfg.run_id(stage), "hub_step": (state or {}).get("step"), "local_step": json.loads(local.read_text()).get("step") if local.exists() else None, }) if args.json: print(json.dumps(rows, indent=2)) else: print(f"{'stage':<9} {'hub':>8} {'local':>8} run_id") for r in rows: print(f"{r['stage']:<9} {str(r['hub_step']):>8} {str(r['local_step']):>8} {r['run_id']}") return 0 def cmd_train(args) -> int: cfg = _cfg(args) from omni.train import execute # Deliberately no run id here: `RunConfig.run_id()` hashes a different identity # key set from `train.get_store()`, and the store's answer is the authoritative # one (it adopts the newest compatible run). `execute` logs the real id. log.info("starting stage %s (preset %s, vocab %s)", args.stage, cfg.preset, cfg.vocab) return execute(cfg, args.stage) def cmd_eval(args) -> int: cfg = _cfg(args) import numpy as np import omni.infer as infer import omni.scenes as scenes system = infer.OmniSystem.load(cfg, token=_token(), log=log.info) rng = np.random.default_rng(args.seed) rows = [] for _ in range(args.scenes): scene = scenes.make_scene(rng) question, truth = scenes.question_answer(scene, rng) image = scenes.render_scene(scene, px=252) pred = system.answer(image, question, max_new=24) rows.append({"q": question, "pred": pred, "truth": truth, "exact": pred.strip() == truth.strip()}) exact = sum(r["exact"] for r in rows) / max(1, len(rows)) print(f"exact match {exact*100:.1f}% over {len(rows)} questions") if args.json: print(json.dumps(rows, indent=2)) return 0 def cmd_movie(args) -> int: cfg = _cfg(args) import numpy as np import omni.infer as infer import omni.scenes as scenes system = infer.OmniSystem.load(cfg, token=_token(), log=log.info) if system.model is None: # no checkpoints on disk: this is an outside user, so use the published weights log.info("no local checkpoints - loading the published weights bundle") system = infer.OmniSystem.from_bundle( getattr(args, "repo", None) or "hkblockchaintechnology/VLM-all-inclusive", token=_token(), log=log.info) if args.image: from PIL import Image image = np.asarray(Image.open(args.image).convert("RGB")) question = args.question or "what do you see" else: rng = np.random.default_rng(args.seed) scene = scenes.make_scene(rng) auto_q, _truth = scenes.question_answer(scene, rng) image = scenes.render_scene(scene, px=252) question = args.question or auto_q result = system.test_to_movie(image, question, out_dir=args.out, log=log.info, size=args.size) print(json.dumps({k: str(v) for k, v in result.items() if k in ("question", "answer", "mp4", "wav", "timeline", "seconds")}, indent=2)) return 0 def cmd_resync(args) -> int: import omni.hub as hub cfg = _cfg(args) pushed = hub.resync_local_runs(cfg.hf_repo, _token(), only_stages=tuple(args.stage) if args.stage else None, include_optim=not args.no_optim) print(f"resynced: {pushed or 'nothing newer than the Hub'}") return 0 def cmd_publish(args) -> int: cfg = _cfg(args) sys.argv = ["publish"] # the publish script parses nothing ns: dict = {"__name__": "omni.cli.publish"} path = Path("/marimo/publish_two_phase.py") if not path.exists(): raise SystemExit("publish_two_phase.py not found next to the notebook") exec(compile(path.read_text(), str(path), "exec"), ns) # noqa: S102 - our own script return 0 def _token() -> str | None: import os tok = os.environ.get("HF_TOKEN") if not tok and Path("/marimo/.hf_token").exists(): tok = Path("/marimo/.hf_token").read_text().strip() return tok or None def build_parser() -> argparse.ArgumentParser: ap = argparse.ArgumentParser("omni", description="OmniVLM: one model, all modalities") ap.add_argument("--config", help="JSON RunConfig to start from") ap.add_argument("--set", action="append", metavar="KEY=VALUE", help="override a config field (JSON value)") ap.add_argument("--verbose", action="store_true") sub = ap.add_subparsers(dest="command", required=True) def add_common(p): p.add_argument("--config", help="JSON RunConfig to start from (overrides --config)") p.add_argument("--set", action="append", metavar="KEY=VALUE", help="override a config field (JSON value)") p.add_argument("--preset", choices=sorted(PRESETS)) p.add_argument("--img_size", type=int) p.add_argument("--data_seed", type=int) p.add_argument("--run_tag") p.add_argument("--hf_repo") p.add_argument("--public_repo") p = sub.add_parser("doctor", help="diagnostics") p.add_argument("--repo") p.add_argument("--json", action="store_true") p.set_defaults(func=cmd_doctor) p = sub.add_parser("stages", help="stage / run id / hub state") add_common(p) p.add_argument("--json", action="store_true") p.set_defaults(func=cmd_stages) p = sub.add_parser("train", help="train or resume a stage") p.add_argument("--stage", required=True, choices=STAGE_NAMES) add_common(p) for flag, kind in (("steps", int), ("batch", int), ("accum", int), ("lr", float), ("log_every", int), ("minutes", float), ("ckpt_steps", int), ("push_seconds", float), ("vl_mix", float)): p.add_argument(f"--{flag}", type=kind) p.add_argument("--resume", choices=("auto", "never")) p.set_defaults(func=cmd_train) p = sub.add_parser("eval", help="exact-match accuracy of the served checkpoint") add_common(p) p.add_argument("--scenes", type=int, default=200) p.add_argument("--seed", type=int, default=2024) p.add_argument("--json", action="store_true") p.set_defaults(func=cmd_eval) p = sub.add_parser("movie", help="image + question -> talking-head mp4") add_common(p) p.add_argument("--seed", type=int, default=7) p.add_argument("--question") p.add_argument("--image") p.add_argument("--out", default="/marimo/out") p.add_argument("--size", type=int, default=384) p.set_defaults(func=cmd_movie) p = sub.add_parser("resync", help="push local checkpoints the Hub is missing") add_common(p) p.add_argument("--stage", action="append", choices=STAGE_NAMES) p.add_argument("--no-optim", action="store_true") p.set_defaults(func=cmd_resync) p = sub.add_parser("publish", help="refresh the public repo") add_common(p) p.set_defaults(func=cmd_publish) return ap def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) configure_logging("DEBUG" if args.verbose else None) try: return int(args.func(args) or 0) except KeyboardInterrupt: # pragma: no cover log.warning("interrupted") return 130 return @app.cell(hide_code=True) def repo_assets(CFG, HF_TOKEN): # --- Restore the engineering assets (tests, scripts, packaging) -------------- # notebook.py and the Hub are the only durable things; /marimo is not. This cell # pulls the repo assets back so `make test` / `python -m omni.cli` work on a # fresh sandbox. It is idempotent - it simply overwrites from the Hub. ASSET_PREFIXES = ("tests/", "scripts/") ASSET_FILES = ("LICENSE", ".gitignore", "pyproject.toml", "Makefile", "conftest.py", "README.md") def restore_repo_assets(repo=None, dest="/marimo/repo", log=print): """Recreate the repo working tree from the published artefacts.""" from huggingface_hub import HfApi, hf_hub_download repo = repo or CFG["public_repo"] api = HfApi(token=HF_TOKEN) wanted = [f for f in api.list_repo_files(repo) if f.startswith(ASSET_PREFIXES) or f in ASSET_FILES] made = [] for path in wanted: cached = hf_hub_download(repo, path, token=HF_TOKEN, force_download=False) out = Path(dest) / path out.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(cached, out) made.append(path) log(f"restored {len(made)} files into {dest} " f"({sum(1 for p in made if p.startswith('tests/'))} tests)") return made restore_btn = mo.ui.run_button(label="restore tests + scripts from the Hub") assets_panel = mo.vstack([ mo.md("### Engineering assets"), mo.md("- `tests/` - the suite (66 tests: renderer fidelity, generator grounding, " "KV-cache equivalence, CTC alignment, resume round-trip, lips, calibration)\n" "- `scripts/` - `fetch_data.sh`, `select_checkpoint.py`, `save_all.py`\n" "- `Makefile`, `pyproject.toml` - `make test`, `make train-all`, `make movie`\n" "- CLI: `python -m omni.cli {doctor|train|eval|movie|resync|publish}`"), restore_btn, ]) return restore_btn, restore_repo_assets @app.cell(hide_code=True) def quality_gate_controls(): # --- Quality gate: run the test suite from the notebook --------------------- import pytest gate_btn = mo.ui.run_button(label="run the test suite") gate_controls = mo.vstack([mo.md("### Quality gate"), mo.hstack([gate_btn])]) return gate_btn, pytest @app.cell(hide_code=True) def quality_gate(gate_btn, pytest): # --- Test results ----------------------------------------------------------- if gate_btn.value: _code = pytest.main(["-q", "--no-header", "-p", "no:cacheprovider", str(Path("/marimo/repo/tests"))]) GATE = {"exit_code": int(_code), "passed": int(_code) == 0} else: GATE = None if GATE: gate_panel = mo.callout( mo.md(f"**{'tests pass' if GATE['passed'] else 'tests FAILED'}** " f"(pytest exit code {GATE['exit_code']}) - see the console for the summary."), kind="success" if GATE["passed"] else "danger") else: gate_panel = mo.md("_press run: pytest over `tests/` (fast, no GPU-heavy fixtures)_") return @app.cell(hide_code=True) def repo_assets_panel(restore_btn, restore_repo_assets): # --- Restore assets action --------------------------------------------------- restored = restore_repo_assets() if restore_btn.value else [] restore_panel = mo.md(f"restored {len(restored)} files: {', '.join(restored[:6])}" + (" ..." if len(restored) > 6 else "")) if restored else \ mo.md("_assets already on disk (they are also in the Hub repo)_") return @app.cell(hide_code=True) def lib_captions( CombinedVocab, OmniVLM, VLMConfig, od, scenes, textmod, zipfile, ): # ===== omni/captions.py ===== real-data stage: real photographs, natural language # v2: keeps the official flickr30k train/val/test split, so a held-out number # means something, and scores BLEU-1..4 against the 5 human references. # Imports come from `setup`; `materialize` writes this back out to # /marimo/omni/captions.py using only the imports this module actually needs. REAL_DIR = Path("/marimo/data/real") THUMBS = REAL_DIR / "thumbs_{size}.npy" INDEX = REAL_DIR / "captions.json" def read_annotations(root: str | Path = REAL_DIR, log=print): """filename -> {'captions': [...], 'split': train|val|test} from the official CSV.""" root = Path(root) out: dict[str, dict] = {} csv_path = root / "flickr_annotations_30k.csv" if not csv_path.exists(): log(f"missing {csv_path}") return out import csv as _csv with open(csv_path, newline="", errors="replace") as fh: for row in _csv.DictReader(fh): fn = (row.get("filename") or "").strip() raw = row.get("raw") or "" try: caps = json.loads(raw) if raw.lstrip().startswith("[") else [raw] except Exception: caps = [raw] caps = [c.strip() for c in caps if isinstance(c, str) and c.strip()] if fn and caps: out[fn] = {"captions": caps[:5], "split": (row.get("split") or "train").strip()} log(f"annotations: {len(out)} images ({sum(len(v['captions']) for v in out.values())} captions)") return out def prepare(root: str | Path = REAL_DIR, size: int = 128, limit: int = 0, log=print): """Decode the archive once into a thumbnail memmap plus a split-carrying index.""" root = Path(root) thumbs_path = Path(str(THUMBS).format(size=size)) if INDEX.exists() and thumbs_path.exists(): rows = json.loads(INDEX.read_text()) log(f"corpus already prepared: {len(rows)} pairs -> {thumbs_path.name}") return len(rows) zip_path = root / "flickr30k-images.zip" if not zip_path.exists(): raise FileNotFoundError(f"missing {zip_path} (run scripts/fetch_real.sh)") ann = read_annotations(root, log=log) names = sorted(ann) if limit: names = names[:limit] from PIL import Image mm = np.lib.format.open_memmap(thumbs_path, mode="w+", dtype=np.uint8, shape=(len(names), size, size, 3)) rows = [] with zipfile.ZipFile(zip_path) as z: members = {Path(i.filename).name: i.filename for i in z.infolist() if i.filename.endswith(".jpg")} for i, name in enumerate(names): member = members.get(name) or members.get(name.replace(" ", "_")) if member is None: continue with z.open(member) as fh: im = Image.open(fh).convert("RGB").resize((size, size), Image.BICUBIC) mm[i] = np.asarray(im, np.uint8) for cap in ann[name]["captions"]: rows.append([i, name, cap, ann[name]["split"]]) if (i + 1) % 5000 == 0: mm.flush() log(f" decoded {i+1}/{len(names)}") mm.flush() del mm INDEX.write_text(json.dumps(rows)) log(f"prepared {len(names)} images, {len(rows)} caption pairs -> {thumbs_path.name}") return len(rows) class FlickrCaptions: """Real photographs with human captions; index-pure and split-aware.""" def __init__(self, size: int = 128, split: str = "train", root: str | Path = REAL_DIR): self.thumbs_path = Path(str(THUMBS).format(size=size)) rows = json.loads(INDEX.read_text()) self.rows = [r for r in rows if split in ("all", r[3])] self.mm = np.load(self.thumbs_path, mmap_mode="r") self.split, self.size = split, size def __len__(self): return len(self.rows) def __getitem__(self, i: int): img_i, _name, caption, _split = self.rows[int(i) % len(self.rows)] return np.asarray(self.mm[img_i]), caption def fetch(self, indices, workers: int = 8): if workers <= 1: return [self[i] for i in indices] import concurrent.futures as cf with cf.ThreadPoolExecutor(max_workers=workers) as ex: return list(ex.map(self.__getitem__, indices)) def collate_captions(vocab, samples, device="cuda", max_len: int = 32): """(image, caption) -> LM batch that predicts the whole caption from .""" seqs = [[vocab.bos] + vocab.words_to_ids(textmod.words_of(c))[:max_len] + [vocab.eos] for _, c in samples] L = max(len(s) for s in seqs) ids = np.full((len(seqs), L), vocab.pad, np.int64) for i, s in enumerate(seqs): ids[i, :len(s)] = s ids = torch.from_numpy(ids) x, y = ids[:, :-1], ids[:, 1:] img = np.stack([im for im, _ in samples]).astype(np.float32) return {"ids": x.to(device), "targets": y.to(device), "mask": (y != vocab.pad).to(device), "images": torch.from_numpy(img / 127.5 - 1.0).permute(0, 3, 1, 2).to(device)} def bleu(pred: str, refs, max_n: int = 4) -> float: """Corpus-style BLEU with add-1 smoothing (4-grams would otherwise collapse to 0).""" from collections import Counter hyp = textmod.words_of(pred) ref_tokens = [textmod.words_of(r) for r in refs] if not hyp or not ref_tokens: return 0.0 scores = [] for n in range(1, max_n + 1): h = Counter(tuple(hyp[i:i + n]) for i in range(len(hyp) - n + 1)) if not h: scores.append(1.0) continue matched = 0 for gram, cnt in h.items(): best = max((Counter(tuple(r[i:i + n]) for i in range(len(r) - n + 1))[gram] for r in ref_tokens), default=0) matched += min(cnt, best) scores.append((matched + 1.0) / (sum(h.values()) + 1.0)) r_len = min((len(r) for r in ref_tokens), key=lambda L: abs(L - len(hyp))) bp = min(1.0, float(np.exp(1 - r_len / max(len(hyp), 1)))) eps = 1e-9 return float(100.0 * bp * np.exp(np.mean([np.log(s + eps) for s in scores]))) @torch.no_grad() def evaluate_captions(model, vocab, ds, device="cuda", n: int = 200, batch: int = 16, prompt=("describe", "the", "image"), log=print): """Held-out BLEU over real photographs the model has never seen.""" model.eval() words = [w for w in prompt if w in vocab.w2i] scores, samples = [], [] for start in range(0, min(n, len(ds)), batch): chunk = [ds[i] for i in range(start, min(start + batch, len(ds)))] imgs = torch.from_numpy(np.stack([c[0] for c in chunk]).astype(np.float32) / 127.5 - 1.0) imgs = imgs.permute(0, 3, 1, 2).to(device) ids = torch.tensor([[vocab.bos] + vocab.words_to_ids(words) + [vocab.sep]] * len(chunk), device=device) out = model.generate_text(ids, images=imgs, max_new=24) for i, (_img, ref) in enumerate(chunk): pred = " ".join(vocab.ids_to_words(out[i])) scores.append(bleu(pred, [ref])) if len(samples) < 3: samples.append((pred, ref)) model.train() mean = float(np.mean(scores)) if scores else 0.0 log(f"held-out caption BLEU-4 over {len(scores)} real photos: {mean:.2f}") for pred, ref in samples: log(f" model: {pred}\n human: {ref}") return {"bleu4": round(mean, 2), "n": len(scores)} def flickr_words(root: str | Path = REAL_DIR, min_count: int = 2, cap: int = 22000): from collections import Counter c = Counter() for v in read_annotations(root, log=lambda *a: None).values(): for caption in v["captions"]: # NB: not `cap` - that is the limit c.update(textmod.words_of(caption)) keep = [(w, n) for w, n in c.items() if n >= min_count] keep.sort(key=lambda kv: (-kv[1], kv[0])) return sorted(w for w, _ in keep[:cap]) def vocab_words_v2(seed: int = 12345, cache="/marimo/runs/shared/vocab_words_v2.json", root: str | Path = REAL_DIR): if Path(cache).exists(): return json.loads(Path(cache).read_text()) extra = sorted(set(od.ljspeech_words()) | set(flickr_words(root))) return od.build_vocab(seed, 4000, extra_words=extra, cache=cache) def make_vocab_v2(seed: int = 12345, cache="/marimo/runs/shared/vocab_words_v2.json"): return CombinedVocab(words=vocab_words_v2(seed, cache), phones=textmod.PHONES) def vocab_identity_v2(seed: int = 12345, cache="/marimo/runs/shared/vocab_words_v2.json"): import hashlib words = vocab_words_v2(seed, cache) return {"vocab_size": len(words), "vocab_hash": hashlib.sha256(json.dumps(words).encode()).hexdigest()[:8], "vocab_cache": cache} def _checkpoint_arch(store, cfg, vocab): """Architecture of a checkpoint, taken from its own `latest/meta.pt`. `latest/model.pt` is a bare state dict -- it has no `meta` key -- so the old `ck["payload"].get("meta", {})` was always empty and the launch config was used instead. That config has been wrong: the real-data run recorded an 8 011-word vocabulary while the model it trained had 16 380 words. `meta.pt` is written from the model object, so it cannot disagree with the weights. A token-space change is refused rather than absorbed: `load_state_dict` would either raise deep inside the load or, with a differently-sized embedding, quietly pair a vocabulary with rows it never trained. """ got = store.load_latest(tag="meta") meta = got["payload"] if got and isinstance(got.get("payload"), dict) else {} n_words = int(meta.get("n_words") or vocab.n_words) n_phones = int(meta.get("n_phones") or vocab.n_phones) if n_words != vocab.n_words: raise RuntimeError( f"{store.run_id}: checkpoint declares {n_words} words but this " f"vocabulary has {vocab.n_words}. Resuming would pair a checkpoint " f"with the wrong embedding table; pass --resume never to start a " f"fresh run on this vocabulary.") return {"model": VLMConfig.from_dict(meta.get("model") or cfg["model"]), "n_words": n_words, "n_phones": n_phones} def stage_captions(cfg, store, log=print, device="cuda"): """Long-horizon real-data training on one trunk: captions + synthetic Q/A.""" import time d = cfg size = d.get("img_size", 128) prepare(size=size, log=log) vocab = make_vocab_v2() train = FlickrCaptions(size=size, split="train") val = FlickrCaptions(size=size, split="val") log(f"stage=captions train={len(train)} val={len(val)} vocab={vocab.n_words} " f"batch={d['batch']} steps={d['steps']}") model, start = None, 0 if d["resume"] != "never": ck = store.load_latest(tag="model") if ck and ck.get("payload"): arch = _checkpoint_arch(store, d, vocab) model = OmniVLM(arch["model"], n_words=arch["n_words"], n_phones=arch["n_phones"]) model.load_state_dict(ck["payload"]) start = ck["step"] log(f"RESUMED {store.run_id} at step {start} on {arch['n_words']} words") if model is None: model = OmniVLM(VLMConfig.from_dict(d["model"]), n_words=vocab.n_words, n_phones=vocab.n_phones) log("fresh real-data trunk on the v2 vocabulary (real photos + speech + scenes)") model = model.to(device).to(memory_format=torch.channels_last) lr0 = d.get("lr", 3e-4) opt = torch.optim.AdamW(model.parameters(), lr=lr0, betas=(0.9, 0.95), weight_decay=0.05, fused=True) if start: ok = store.load_latest(tag="optim") if ok and ok.get("payload"): from omni.train_vlm import load_opt_flat load_opt_flat(opt, ok["payload"]) amp = torch.autocast("cuda", dtype=torch.bfloat16) warm = d.get("warmup", 500) sched = lambda s: min(1.0, (s + 1) / warm) * (0.5 * (1 + np.cos(np.pi * s / d["steps"]))) t0, step, losses = time.time(), start, [] while step < d["steps"]: real = np.random.default_rng(step).random() >= d.get("scene_mix", 0.3) if real: idx = od.batch_indices_for_step(step, d["batch"], d["data_seed"] + 991) batch = collate_captions(vocab, train.fetch(idx, workers=d.get("workers", 4)), device=device, max_len=d.get("max_len", 32)) else: spec = scenes.spec_batch(od.batch_indices_for_step(step, d["batch"], d["data_seed"]), seed=d["data_seed"], px=size) batch = od.collate_specs(vocab, spec, device=device, px=size) opt.zero_grad(set_to_none=True) with amp: out = model.forward_text(batch["ids"], images=batch["images"], targets=batch["targets"], loss_mask=batch.get("mask")) loss = out["loss"] loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), d.get("clip", 1.0)) for g in opt.param_groups: g["lr"] = lr0 * sched(step) opt.step() losses.append(float(loss.detach())) step += 1 if step % d.get("log_every", 50) == 0: sps = (time.time() - t0) / max(1, step - start) store.log(stage="captions", step=step, loss=float(np.mean(losses[-d.get("log_every", 50):])), acc=float(out.get("acc", 0)), lr=opt.param_groups[0]["lr"], sec_per_step=round(sps, 3), kind="real" if real else "scene") log(f"captions step {step:7d} loss {np.mean(losses[-d.get('log_every',50):]):.4f} " f"acc {float(out.get('acc', 0)):.3f} {sps*1000:.0f} ms/step " f"({'real photo' if real else 'scene'})") if store.should_checkpoint(step) or step >= d["steps"]: store.save(stage="captions", step=step, payload={"model": model.state_dict(), "optim": _flat(opt), "meta": {"model": model.cfg.to_dict(), "n_words": vocab.n_words, "n_phones": vocab.n_phones, "kind": "omni-real"}}, metrics={"loss": float(np.mean(losses[-50:]))}) if len(val): store.log(stage="captions", step=step, **evaluate_captions(model, vocab, val, device=device, n=64, log=log)) if d.get("minutes") and (time.time() - t0) > d["minutes"] * 60: log(f"time budget reached at step {step}") break return model, step def _flat(opt): from omni.train_vlm import _opt_flat return _opt_flat(opt) return @app.cell(hide_code=True) def lib_supervise(): # ===== omni/supervise.py ===== the autopilot (long-horizon job runner) # Imports come from `setup`; `materialize` writes this back out to /marimo/omni/supervise.py # ============================================================================= # omni/supervise.py - the autopilot. # # Purpose: keep the GPU busy for days on a box that can die at any moment. # # * runs a QUEUE of jobs (stage + step budget), sequentially # * every job is invoked through the normal resume path, so a crashed or # killed job simply continues from its last checkpoint # * on exit of any job: force a full push (model + optimizer + state) to the # Hub, so nothing is only on the ephemeral disk # * writes runs/supervisor.json after every transition: the notebook reads it # to show live status, and it survives a restart # * --max-hours stops cleanly (flush + exit) so a scheduled restart costs # one job handover, not the run # # Usage: # python -m omni.supervise --job captions:120000 --job speech:8000 \ # --job vocoder:12000 --job ear:12000 --loop # ============================================================================= STATUS = Path("/marimo/runs/supervisor.json") SUPERVISOR_LOG_DIR = Path("/marimo/logs") def _write_status(payload, log=lambda *a: None): STATUS.parent.mkdir(parents=True, exist_ok=True) payload["updated"] = time.time() payload["updated_at"] = time.strftime("%H:%M:%S") tmp = STATUS.with_suffix(".json.tmp") tmp.write_text(json.dumps(payload, indent=1, default=str)) os.replace(tmp, STATUS) return payload def read_status(): try: return json.loads(STATUS.read_text()) except Exception: return {} def _step_of(stage_run_prefix: str | None, root="/marimo/runs"): """Latest step recorded by the trainer for a run id prefix.""" if not stage_run_prefix: return 0 best = 0 for d in Path(root).glob(f"{stage_run_prefix}*"): f = d / "log.jsonl" if f.exists(): for line in f.read_text().splitlines()[-1:]: try: best = max(best, int(json.loads(line).get("step", 0))) except Exception: pass return best def _resync(only_stages, log): try: from omni.hub import resync_local_runs tok = os.environ.get("HF_TOKEN") or ( Path("/marimo/.hf_token").read_text().strip() if Path("/marimo/.hf_token").exists() else None) pushed = resync_local_runs("hkblockchaintechnology/omni-vlm-ckpt", tok, retries=2, include_optim=True, only_stages=tuple(only_stages)) log(f"[supervisor] pushed to the Hub: {pushed}") return pushed except Exception as e: # never let this kill the loop log(f"[supervisor] resync failed: {type(e).__name__} {e}") return [] def run(jobs, preset="xl", batch=None, lr=None, loop=False, max_hours=0.0, backoff=45.0, log=print, dry_run=False, max_retries: int = 3): """jobs: list of (stage, steps, extra_config_dict).""" t_start = time.time() status = read_status() or {"created": time.time(), "jobs": [], "restarts": 0, "runs": 0} status["jobs"] = [{"stage": s, "budget": n, "extra": x} for s, n, x in jobs] status["pid"] = os.getpid() status["log"] = str(SUPERVISOR_LOG_DIR / "supervisor.log") _write_status(status, log) stages = {s for s, _, _ in jobs} def _stop(signum, frame): log(f"[supervisor] signal {signum}: flushing and exiting") status["stopped"] = time.strftime("%H:%M:%S") _write_status(status, log) _resync(stages, log) raise SystemExit(0) for sig in (signal.SIGTERM, signal.SIGINT): try: signal.signal(sig, _stop) except Exception: pass while True: i = 0 while i < len(jobs): stage, budget, extra = jobs[i] i += 1 # go through the CLI: it owns the RunConfig plumbing, so the # autopilot does not depend on module entry points staying put args = ["python", "-m", "omni.cli", "train", "--stage", stage, "--preset", preset, "--steps", str(budget), "--run_tag", "main"] if batch: args += ["--batch", str(batch)] if lr: args += ["--lr", str(lr)] for k, v in extra.items(): args += ["--set", f"{k}={json.dumps(v)}"] tail = SUPERVISOR_LOG_DIR / f"{stage}.log" log(f"[supervisor] === {stage}: budget {budget} steps " f"(run {status.get('runs', 0) + 1}) ===") status.update({"current": stage, "current_index": i, "started": time.time(), "started_at": time.strftime("%H:%M:%S"), "runs": status.get("runs", 0) + 1}) _write_status(status, log) if dry_run: log("[supervisor] dry run - would execute: " + " ".join(args)) continue t0 = time.time() with open(tail, "a") as logf: logf.write(f"\n===== supervisor run {time.strftime('%F %T')} " f"{' '.join(args)} =====\n") proc = subprocess.Popen(args, cwd="/marimo", stdout=logf, stderr=subprocess.STDOUT, start_new_session=True) rc = proc.wait() done = _step_of(f"{stage}-{preset}") status.update({"last_stage": stage, "last_exit": rc, "last_seconds": round(time.time() - t0, 1), "last_step": done, "last_hours": round((time.time() - t0) / 3600, 2)}) log(f"[supervisor] {stage} exited rc={rc} after {(time.time()-t0)/3600:.2f} h " f"at step {done}/{budget}") _write_status(status, log) _resync([stage], log) if rc != 0: attempts = status.setdefault("attempts", {}) attempts[stage] = attempts.get(stage, 0) + 1 status["restarts"] = status.get("restarts", 0) + 1 _write_status(status, log) if attempts[stage] <= max_retries: log(f"[supervisor] {stage} failed (attempt {attempts[stage]}/{max_retries}): " f"retrying in {backoff:.0f}s - it resumes from its last checkpoint") time.sleep(backoff) jobs.insert(i - 1, (stage, budget, extra)) # retry THIS job next else: log(f"[supervisor] {stage} failed {attempts[stage]} times: moving on") attempts[stage] = 0 if max_hours and (time.time() - t_start) / 3600 >= max_hours: log(f"[supervisor] --max-hours {max_hours} reached: flushing and exiting") _resync(stages, log) status["stopped"] = "max-hours" _write_status(status, log) return status if not loop: break log("[supervisor] queue finished; --loop is on, starting over") status["stopped"] = "done" _write_status(status, log) return status def parse_jobs(specs): jobs = [] for spec in specs: parts = spec.split(":") stage = parts[0] steps = int(parts[1]) if len(parts) > 1 and parts[1] else 100000 extra = {} for kv in parts[2:]: k, _, v = kv.partition("=") try: extra[k] = json.loads(v) except Exception: extra[k] = v jobs.append((stage, steps, extra)) return jobs def supervisor_main(argv=None): import argparse ap = argparse.ArgumentParser("omni.supervise") ap.add_argument("--job", action="append", default=[], help="stage:steps[:key=json], repeatable") ap.add_argument("--preset", default="xl") ap.add_argument("--batch", type=int) ap.add_argument("--lr", type=float) ap.add_argument("--loop", action="store_true", help="start the queue over when finished") ap.add_argument("--max-hours", type=float, default=0.0) ap.add_argument("--backoff", type=float, default=45.0) ap.add_argument("--dry-run", action="store_true") args = ap.parse_args(argv) jobs = parse_jobs(args.job) or [("captions", 100000, {})] stat = run(jobs, preset=args.preset, batch=args.batch, lr=args.lr, loop=args.loop, max_hours=args.max_hours, backoff=args.backoff, log=lambda m: print(m, flush=True), dry_run=args.dry_run) print(json.dumps({k: stat.get(k) for k in ("runs", "restarts", "last_stage", "last_step")}, indent=1)) return 0 raise SystemExit(supervisor_main()) return if __name__ == "__main__": app.run()