chaoliangUNSW's picture
Jev-Style-2B-Decision-v4: weights, runtime, reference code, card
3f2c063 verified
Raw History Blame Contribute Delete
12.8 kB
"""Pool access and micro-batch collation for training and dev scoring.
Rows follow CONTRACTS section 7 (``macjev.pool.load``). Token lists inside ``pieces`` are
compacted to int32 numpy arrays after loading (a Python int list costs ~36 bytes/token)
and turned back into lists only when ``Renderer.assemble`` needs them.
A *sequence* is one rendered view of a row: ids, readout slots, and the row's targets
permuted into *canonical* option order is never needed because scores are mapped back
to canonical order inside the model (see ``model.forward_scores``).
"""
from __future__ import annotations
import json
import math
import re
from collections import Counter
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterable, Sequence
import numpy as np
import torch
from ..render import CONTEXT_LIMIT, InputBudgetError, Renderer
from ..schema import TRAINABLE_LICENSE_PREFIXES
TOKEN_KEYS = ("prefix", "head")
# same patterns as configs/macjev/quotas.json assembly.nc_regexes (assemble enforces them; this is a second line)
NC_PATTERNS = (r"(^|[^a-z])nc([^a-z]|$)", r"non[- ]?commercial", r"research[- ]?(only|use|purposes)",
r"no[- ]commercial")
# ----------------------------------------------------------------------------- loading
def load_pool_rows(tok_tag: str, split: str, pool_dir: str | Path | None = None) -> list[dict]:
"""``macjev.pool.load`` (the only pool reader) unless a fixture directory is given."""
if pool_dir is not None and (Path(pool_dir) / tok_tag / "manifest.json").exists():
from .. import pool # a real pool stored under another root (e.g. a synced volume)
return [compact(r) for r in pool.open_split(tok_tag, split, root=pool_dir)]
if pool_dir is not None:
path = Path(pool_dir) / f"{split}.jsonl"
if not path.exists():
return []
with path.open() as f:
# Compact each row immediately; never materialize a whole epoch as Python ints.
return [compact(json.loads(line)) for line in f if line.strip()]
from .. import pool # implemented by the assemble package (CONTRACTS section 7)
if hasattr(pool, "open_split"):
# stream rows and compact each one at once: peak memory ~4 bytes/token instead of a list of ints
return [compact(r) for r in pool.open_split(tok_tag, split)]
return pool.load(tok_tag, split)
def compact(row: dict) -> dict:
p = row["pieces"]
if not isinstance(p.get("prefix"), np.ndarray):
p = dict(p)
for k in TOKEN_KEYS:
p[k] = np.asarray(p[k], dtype=np.int32)
p["opts"] = [np.asarray(o, dtype=np.int32) for o in p["opts"]]
row = dict(row, pieces=p)
return row
def pieces_lists(p: dict) -> dict:
out = dict(p)
for k in TOKEN_KEYS:
out[k] = p[k].tolist() if isinstance(p[k], np.ndarray) else list(p[k])
out["opts"] = [o.tolist() if isinstance(o, np.ndarray) else list(o) for o in p["opts"]]
return out
def row_len(row: dict, readout: str) -> int | None:
n = row.get("n_tokens", {}).get(readout)
return None if n is None else int(n)
def row_weight(row: dict) -> float:
w=float(row.get("weight",1.0))
if not math.isfinite(w) or w<=0:raise ValueError(f"Invalid sample weight for {row.get('id')}")
return w
class LicenceError(ValueError):
pass
def check_licence(row: dict) -> None:
"""CONTRACTS section 0: NC / research-only / unknown-restrictive data may only be split "test",
and test rows are never pooled. A violation means the pool is broken: fail loudly, never skip."""
if row.get("split") == "test":
raise LicenceError(f"row {row['id']}: split 'test' rows must never reach training or dev")
lic = str(row.get("license", "")).lower()
if not lic.startswith(TRAINABLE_LICENSE_PREFIXES) or any(re.search(p, lic) for p in NC_PATTERNS):
raise LicenceError(f"row {row['id']} ({row.get('source')}): licence {row.get('license')!r} is evaluation-only")
def check_row(row: dict) -> None:
check_licence(row)
row_weight(row)
names = row["names"]
k = len(names)
if list(row["pieces"]["names"]) != list(names):
raise ValueError(f"row {row['id']}: pieces names differ from row names")
if len(row["pieces"]["opts"]) != k:
raise ValueError(f"row {row['id']}: {len(row['pieces']['opts'])} option pieces for {k} names")
if not row["valid_idx"] or not all(0 <= i < k for i in row["valid_idx"]):
raise ValueError(f"row {row['id']}: bad valid_idx")
if row.get("soft") is not None:
s = np.asarray(row["soft"], dtype=np.float64)
if s.shape != (k,) or (s < 0).any() or abs(s.sum() - 1) > 1e-3:
raise ValueError(f"row {row['id']}: soft must be a distribution aligned to names")
if row["qtype"] == "score" and len(row["valid_idx"]) != 1:
raise ValueError(f"row {row['id']}: score rows need exactly one valid level")
@dataclass
class Selection:
rows: list[dict]
dropped: Counter = field(default_factory=Counter)
drop_source_hits: dict = field(default_factory=dict)
def select_rows(rows: Iterable[dict], readout: str, max_len: int, categories: Sequence[str] | None = None,
drop_sources: Sequence[str] = (), max_rows: int | None = None, seed: int = 0) -> Selection:
"""Filter rows for training; every exclusion is counted (never silent)."""
rows = list(rows)
if max_rows is not None and len(rows) > max_rows:
rng = np.random.default_rng(seed)
keep = sorted(rng.choice(len(rows), size=max_rows, replace=False).tolist())
rows = [rows[i] for i in keep]
cats = set(categories) if categories else None
drops = set(drop_sources)
hits = {s: 0 for s in drops}
out, dropped = [], Counter()
for r in rows:
if r["source"] in drops:
hits[r["source"]] += 1
dropped["ablation_drop_source"] += 1
continue
if cats is not None and r["category"] not in cats:
dropped["category_not_selected"] += 1
continue
n = row_len(r, readout)
if n is None:
dropped[f"no_{readout}_rendering"] += 1
continue
if n > max_len:
dropped[f"longer_than_{max_len}"] += 1
continue
check_row(r)
out.append(compact(r))
return Selection(out, dropped, hits)
# ----------------------------------------------------------------------------- sequences
@dataclass
class Seq:
row: int # index into the row list
ids: list[int]
slots: list[int]
order: list[int] # rendered position -> canonical index
pair: int = -1 # >=0: index of the other view of the same row in the same micro-batch
blocks: list | None = None
def assemble_seq(renderer: Renderer, row: dict, row_idx: int, order: Sequence[int] | None) -> Seq:
pieces = pieces_lists(row["pieces"])
if hasattr(renderer, "layout"):
pieces["layout"] = row.get("render_layout", renderer.layout)
rendered = renderer.assemble(pieces, order, head_max=row.get("head_max"))
expect = row_len(row, renderer.readout)
mismatch = len(rendered.ids) > expect if expect is not None and hasattr(renderer, "layout") else len(rendered.ids) != expect
if expect is not None and mismatch:
raise InputBudgetError(f"row {row['id']}: assembled {len(rendered.ids)} tokens, pool says {expect}")
return Seq(row=row_idx, ids=rendered.ids, slots=rendered.slots, order=rendered.order,
blocks=rendered.meta.get("blocks"))
@dataclass
class Batch:
input_ids: torch.Tensor # [B, T] right padded
attention_mask: torch.Tensor # [B, T] long
slot_pos: torch.Tensor # [B, S] verdict: one per rendered option; letter: [B, 1]
slot_mask: torch.Tensor # [B, S] bool
order: torch.Tensor # [B, K] rendered pos -> canonical index, padded with identity
n_options: torch.Tensor # [B]
valid: torch.Tensor # [B, K] bool, canonical order
soft: torch.Tensor # [B, K] float32 canonical (zeros where absent)
has_soft: torch.Tensor # [B] bool
is_score: torch.Tensor # [B] bool
pair: torch.Tensor # [B] long, -1 if unpaired
rows: list[int]
real_tokens: int
padded_tokens: int
blocks: list | None = None
lengths: list | None = None
weights: torch.Tensor | None = None
pixel_values: torch.Tensor | None = None # v4: [sum patches, C] for every image of the batch, row-major
image_grid_thw: torch.Tensor | None = None # v4: [n_images, 3]
mm_token_type_ids: torch.Tensor | None = None # v4: [B, T] 1 at image-pad positions
def to(self, device) -> "Batch":
kw = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in self.__dict__.items()}
return Batch(**kw)
def collate(seqs: Sequence[Seq], rows: Sequence[dict], readout: str, pad_id: int,
image_loader=None, image_pad_id: int | None = None) -> Batch:
"""``image_loader(path) -> (pixel_values [n_patches, C], grid_thw [3])`` (v4, ``macjev.vision.ImageLoader``);
rows whose ``pieces["images"]`` is non-empty get their tensors concatenated in batch-row order."""
B = len(seqs)
T = max(len(s.ids) for s in seqs)
K = max(len(rows[s.row]["names"]) for s in seqs)
S = K if readout == "verdict" else 1
ids = torch.full((B, T), pad_id, dtype=torch.long)
am = torch.zeros((B, T), dtype=torch.long)
slot_pos = torch.zeros((B, S), dtype=torch.long)
slot_mask = torch.zeros((B, S), dtype=torch.bool)
order = torch.arange(K).repeat(B, 1)
n_opt = torch.zeros(B, dtype=torch.long)
valid = torch.zeros((B, K), dtype=torch.bool)
soft = torch.zeros((B, K), dtype=torch.float32)
has_soft = torch.zeros(B, dtype=torch.bool)
is_score = torch.zeros(B, dtype=torch.bool)
pair = torch.tensor([s.pair for s in seqs], dtype=torch.long)
real = 0
for b, s in enumerate(seqs):
r = rows[s.row]
k = len(r["names"])
n = len(s.ids)
real += n
ids[b, :n] = torch.tensor(s.ids, dtype=torch.long)
am[b, :n] = 1
slot_pos[b, :len(s.slots)] = torch.tensor(s.slots, dtype=torch.long)
slot_mask[b, :len(s.slots)] = True
order[b, :k] = torch.tensor(s.order, dtype=torch.long)
n_opt[b] = k
valid[b, list(r["valid_idx"])] = True
if r.get("soft") is not None:
soft[b, :k] = torch.tensor(r["soft"], dtype=torch.float32)
has_soft[b] = True
is_score[b] = r["qtype"] == "score"
batch = Batch(ids, am, slot_pos, slot_mask, order, n_opt, valid, soft, has_soft, is_score, pair,
[s.row for s in seqs], real, B * T, [s.blocks for s in seqs], [len(s.ids) for s in seqs],
torch.tensor([row_weight(rows[s.row]) for s in seqs],dtype=torch.float32))
images = [(b, img) for b, s in enumerate(seqs) for img in (rows[s.row]["pieces"].get("images") or [])]
if images:
if image_loader is None or image_pad_id is None:
raise ValueError("batch has image rows but no image_loader/image_pad_id was given")
pv, thw = [], []
for b, img in images:
x, g = image_loader(img["path"])
pv.append(x); thw.append(torch.as_tensor(g, dtype=torch.long).reshape(3))
n_expected = int(img["n_tokens"])
if int(torch.as_tensor(g).prod()) // int(img.get("merge", 2)) ** 2 != n_expected:
raise ValueError(f"image {img['path']}: {n_expected} pad tokens in the pool but the loader produced grid {g}")
batch.pixel_values = torch.cat(pv, 0)
batch.image_grid_thw = torch.stack(thw, 0)
batch.mm_token_type_ids = (ids == image_pad_id).long()
n_pads = int(batch.mm_token_type_ids.sum())
# each image's own grid (10-25 fix: the old generator reused the loop variable g = the LAST image's grid,
# so a batch with two differently sized images failed this check although the batch was correct)
n_feat = int(sum(int(t.prod()) // int(img.get("merge", 2)) ** 2 for t, (_, img) in zip(thw, images)))
if n_pads != n_feat:
raise ValueError(f"image pads {n_pads} != image features {n_feat}")
return batch
def length_bucket(n: int, bounds: Sequence[int]) -> int:
for i, b in enumerate(bounds):
if n <= b:
return i
raise InputBudgetError(f"{n} tokens exceed the largest bucket {bounds[-1]}")
EXPOSURE_BINS = (512, 1024, 2048, 4096, 8192, 16384, CONTEXT_LIMIT)
def bucket_label(n: int) -> str:
for b in EXPOSURE_BINS:
if n <= b:
return f"<={b}"
return f">{EXPOSURE_BINS[-1]}"