import { Arrow, FigBox, FILL, LoopFigure, type LoopStop } from "./LoopFigure"; const H = [ { y: 110, label: "h₁" }, { y: 190, label: "h₂" }, { y: 270, label: "h₃" }, { y: 350, label: "h₄" }, ]; const STOPS: LoopStop[] = [ { x: 120, y: 190, ids: ["score"], inspect: "score", title: "Encoder states", note: "Every source position keeps its hidden state. Attention will read from all of them.", }, { x: 520, y: 90, ids: ["score"], inspect: "score", title: "Decoder state s_t", note: "The query: how relevant is each encoder state to the token being generated now?", }, { x: 330, y: 200, ids: ["score"], inspect: "score", title: "Score(s_t, h_i)", note: "A small net (or a dot product) measures alignment between the decoder and each source position.", }, { x: 330, y: 300, ids: ["weights"], inspect: "weights", title: "Softmax → α_i", note: "Scores become a distribution over the source. That heatmap is the attention.", }, { x: 520, y: 250, ids: ["context"], inspect: "context", title: "Context c_t", note: "c_t = Σ α_i h_i. A dynamic memory read — no single bottleneck vector.", }, { x: 520, y: 360, ids: ["context"], inspect: "context", title: "Next token", note: "The decoder uses c_t (and s_t) to emit the next symbol, then the loop advances t.", }, { x: 520, y: 410, ids: ["context"], inspect: "context", title: "Advance t", note: "The decoder steps forward and queries the same encoder memory again.", }, { x: 120, y: 410, ids: ["score"], inspect: "score", title: "Back to source", note: "Encoder states stay; only the decoder time index changes.", }, ]; export function AttentionLoop({ onSelect }: { onSelect: (id: string) => void }) { return ( {({ on, markerId }) => ( Attention (Bahdanau 2014) c_t = Σ softmax(score(s_t, h_i)) h_i Encoder {H.map((h) => ( onSelect("score")} /> ))} onSelect("score")} /> onSelect("weights")} /> onSelect("score")} /> onSelect("context")} /> onSelect("context")} /> Source positions Target step t )} ); }