Blame for locus/models/draft_verify.py

"""Box head as a speculative drafter for coordinate tokens (Jacobi verification over a shared prefix).

The box head answers every question of an image in one branch pass over the cached image prefix. Its boxes are
rendered as the backbone's own coordinate-token answer ("x0, y0, x1, y1]" in 0–1000 units, after a teacher-forced
JSON preamble) and verified in parallel by the backbone's LM head, with the adapter disabled (base model):

  pass k:  argmax g_t of every draft position given the draft before it, all questions batched as branches.
           Accept the longest prefix with g_t == draft_t; the token after it, g_n, is exact too.
           Next draft = accepted + g_n + g_{n+1..} (Jacobi iterate). Stop when nothing changes.

The fixed point is exactly the greedy decode under the same computation. Rows still unconverged after max_passes
finish with plain greedy decoding from their accepted prefix, so the output is always the greedy answer; the cost
is the number of sequential passes. max_passes=0 is plain greedy decoding (the coordinate-token control on the same
prompt, cache and kernels).

accept < 1 relaxes the check (typical acceptance, as in Medusa): a draft token also passes when the verifier gives it
at least accept x the probability of its argmax. The output is then no longer the greedy answer but the head's box
unless the verifier clearly disagrees; accept=1 is the exact rule above.
"""
from __future__ import annotations

import re

import numpy as np
import torch

from locus.models.boxvlm import BOX_TOKEN, BoxVLM

# Continues BoxVLM.prefix ("...Locate:"): the zero-shot grounding prompt of the Qwen harness, with a colon.
VERIFY_PROMPT = ' "{}" in the image and output its bbox coordinates in JSON format.'
PREAMBLE = '```json\n[\n\t{"bbox_2d": ['


class DraftVerify:
    def __init__(self, model: BoxVLM, max_passes: int = 4, max_tokens: int = 24, verifier_adapter: bool = False,
                 accept: float = 1.0):
        assert model.n_layers_run == model.n_layers_total, "the verifier needs the full LM (no early exit)"
        self.m, self.tok = model, model.ad.tok
        self.max_passes, self.max_tokens = max_passes, max_tokens
        self.close = self.tok.encode("]", add_special_tokens=False)[0]
        self.vsuffix = model.suffix.replace(BOX_TOKEN, "") + PREAMBLE
        self.verifier_adapter = verifier_adapter  # aLoRA trained with --ar-weight: adapter on verifier branches too
        self.log_accept = float(np.log(accept)) if accept < 1 else None
        self._closes = {}

    # -- token helpers ------------------------------------------------------
    def is_close(self, t: int) -> bool:
        if t not in self._closes:
            self._closes[t] = self.tok.decode([t]).lstrip().startswith("]")
        return self._closes[t]

    def cut(self, ids: list[int]) -> list[int]:
        """Truncate after the first token that closes the coordinate list; append "]" if there is none."""
        for i, t in enumerate(ids):
            if self.is_close(t):
                return ids[: i + 1]
        return ids + [self.close]

    def render(self, box) -> list[int]:
        return self.tok.encode(", ".join(str(int(round(min(max(float(v), 0.0), 1.0) * 1000))) for v in box) + "]",
                               add_special_tokens=False)

    def parse(self, ids: list[int]) -> np.ndarray:
        m = re.match(r"\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\]", self.tok.decode(ids))
        if m is None:
            return np.full(4, np.nan)
        x0, y0, x1, y1 = (min(int(v), 1000) / 1000 for v in m.groups())
        return np.array([min(x0, x1), min(y0, y1), max(x0, x1), max(y0, y1)])

    def verifier_inputs(self, texts, boxes=None):
        """Verifier-branch prompt tokens per question (continuing BoxVLM.prefix), and rendered answers if boxes given."""
        prompts = [self.tok.encode(VERIFY_PROMPT.format(t) + self.vsuffix, add_special_tokens=False) for t in texts]
        return prompts, None if boxes is None else [self.render(b) for b in boxes]

    # -- verifier passes ----------------------------------------------------
    def _greedy_at(self, state, prompts, drafts, rows, row_image):
        """One batched branch pass: argmax at every draft position (and the one after the last) for the given rows.
        With accept < 1, a draft token the verifier finds plausible enough is returned in place of the argmax."""
        seqs = [prompts[i] + drafts[i] for i in rows]
        L = max(map(len, seqs))
        ids = torch.full((len(seqs), L), self.tok.pad_token_id, dtype=torch.long)
        mask = torch.zeros(len(seqs), L, dtype=torch.long)
        for j, s in enumerate(seqs):
            ids[j, : len(s)], mask[j, : len(s)] = torch.tensor(s), 1
        dev = row_image.device
        if self.m.fast is not None:
            h = self.m.branch_pass(state, ids.to(dev), mask.to(dev), row_image[rows], keep_cache=True,
                                   adapter=self.verifier_adapter).last_hidden_state
        else:
            ctx = self.m.ad.model.disable_adapter() if self.m.lora_r and not self.verifier_adapter else torch.no_grad()
            with ctx:
                h = self.m.branch_pass(state, ids.to(dev), mask.to(dev), row_image[rows], keep_cache=True).last_hidden_state
        spans = [(j, len(prompts[i]) - 1, len(prompts[i]) + len(drafts[i])) for j, i in enumerate(rows)]
        sel = torch.cat([h[j, a:b] for j, a, b in spans])  # LM head only where needed: len(draft) + 1 per row
        logits = self.m.base.lm_head(sel).float()
        pred = logits.argmax(-1)
        if self.log_accept is not None:
            draft = torch.tensor([t for i in rows for t in drafts[i] + [-1]], device=pred.device)
            lp = logits.log_softmax(-1)
            ok = (draft >= 0) & (lp.gather(1, draft.clamp(min=0)[:, None])[:, 0] - lp.max(-1).values >= self.log_accept)
            pred = torch.where(ok, draft, pred)
        pred = pred.tolist()
        out, k = [], 0
        for _, a, b in spans:
            out.append(pred[k : k + b - a])
            k += b - a
        return out

    @torch.no_grad()
    def __call__(self, prefix_inputs, texts: list[str], row_image: torch.Tensor, verify: bool = True,
                 draft: np.ndarray | None = None) -> dict:
        """`draft` (boxes from another model's head) replaces this model's head as the drafter."""
        m = self.m
        state = m.prefill(prefix_inputs)
        bid, bmask = m.tokenize_branches(texts)
        dev = row_image.device
        head, conf = m.forward_shared(None, bid.to(dev), bmask.to(dev), row_image, state=state)
        head = head.float().cpu().numpy()
        res = dict(head=head, conf=torch.sigmoid(conf.float()).cpu().numpy())
        if not verify:
            return res
        assert m.alora or not m.lora_r or self.verifier_adapter, \
            "the base-model verifier needs a base prefix: use an aLoRA (or frozen) head, or a jointly trained adapter"
        prompts = self.verifier_inputs(texts)[0]
        R = len(texts)
        drafts = [self.render(b) for b in (head if draft is None else draft)] if self.max_passes else [[] for _ in range(R)]
        acc = [0] * R  # length of the greedy-exact prefix of each draft
        passes = np.zeros(R, int)
        active = list(range(R)) if self.max_passes else []
        for _ in range(self.max_passes):
            if not active:
                break
            g = self._greedy_at(state, prompts, drafts, active, row_image)
            nxt = []
            for i, gi in zip(active, g):
                passes[i] += 1
                d = drafts[i]
                n = next((k for k, (a, b) in enumerate(zip(d, gi)) if a != b and not (self.is_close(a) and self.is_close(b))), len(d))
                if n == len(d):  # fixed point: every draft token is the greedy token
                    acc[i] = len(d)
                    continue
                acc[i] = n + 1  # g_n is exact as well
                drafts[i] = self.cut(d[:n] + gi[n:])
                nxt.append(i)
            active = nxt
        # Unconverged rows (and max_passes=0): plain greedy decoding from the exact prefix.
        todo = [i for i in range(R) if acc[i] < len(drafts[i]) or not drafts[i]]
        for i in todo:
            drafts[i] = drafts[i][: acc[i]]
        while todo:
            g = self._greedy_at(state, prompts, [d for d in drafts], todo, row_image)
            nxt = []
            for i, gi in zip(todo, g):
                passes[i] += 1
                drafts[i] = drafts[i] + [gi[-1]]
                if not self.is_close(gi[-1]) and len(drafts[i]) < self.max_tokens:
                    nxt.append(i)
            todo = nxt
        res.update(box=np.stack([self.parse(d) for d in drafts]), passes=passes, tokens=np.array([len(d) for d in drafts]))
        return res