import { Arrow, FigBox, FILL, LoopFigure, type LoopStop } from "./LoopFigure"; const STOPS: LoopStop[] = [ { x: 160, y: 110, ids: ["vision"], inspect: "vision", title: "Image patches", note: "The image is cut into patches. This is the new front-end; the language model still has not seen pixels.", }, { x: 160, y: 170, ids: ["vision"], inspect: "vision", title: "ViT encoder", note: "A vision Transformer (often CLIP-ViT) turns patches into a sequence of embeddings.", }, { x: 160, y: 270, ids: ["projector"], inspect: "projector", title: "Projector W", note: "A linear layer or small MLP maps vision width onto the LLM hidden size.", }, { x: 500, y: 170, ids: ["decoder"], inspect: "decoder", title: "Text tokens", note: "The prompt is embedded as usual. Vision tokens will sit in the same residual stream.", }, { x: 330, y: 360, ids: ["decoder", "projector"], inspect: "decoder", title: "Concat [vision ; text]", note: "z = LLM([W · ViT(image); text]). Cross-attention variants keep the ViT as a separate memory.", }, { x: 330, y: 450, ids: ["decoder"], inspect: "decoder", title: "LLM decoder → logits", note: "The top layer is still next-token prediction. Once vision tokens are in context, the same stack can caption or follow instructions.", }, ]; export function VlmLoop({ onSelect }: { onSelect: (id: string) => void }) { return ( {({ on, markerId }) => ( Vision-Language Model z = LLM([W · ViT(image); text]) Vision {[0, 1, 2].map((r) => [0, 1, 2].map((c) => ( )), )} onSelect("vision")} /> onSelect("projector")} /> Language onSelect("decoder")} /> onSelect("decoder")} /> onSelect("decoder")} /> onSelect("decoder")} /> )} ); }