VLM-all-inclusive / tests /test_data.py
hkblockchaintechnology's picture
repo assets: tests, scripts, Makefile, packaging, refreshed card
080b493 verified
Raw History Blame Contribute Delete
4.06 kB
"""Data invariants: the renderer must match the spec, and batching must be pure.
These are the properties the whole training pipeline rests on - if the renderer
drifts from the scene spec, every grounded answer becomes a lie.
"""
import numpy as np
import pytest
import torch
import omni.render as render
import omni.scenes as scenes
def test_scene_spec_is_self_consistent():
rng = np.random.default_rng(0)
for _ in range(30):
sc = scenes.make_scene(rng)
assert 1 <= len(sc["objects"]) <= 4
cells = [o["cell"] for o in sc["objects"]]
assert len(set(cells)) == len(cells), "one object per grid cell"
for o in sc["objects"]:
assert o["shape"] in scenes.SHAPES
assert o["color"] in scenes.COLORS
assert 0.15 <= o["cx"] <= 0.85 and 0.15 <= o["cy"] <= 0.85
def test_numpy_renderer_draws_the_spec():
"""The pixel at each object centre must carry that object's colour."""
for seed in range(12):
img, _, _, sc = scenes.render_and_ask(seed, seed)
for o in sc["objects"]:
x, y = int(o["cx"] * 128), int(o["cy"] * 128)
got = img[y, x].astype(int)
want = np.array(scenes.COLORS[o["color"]])
assert np.abs(got - want).max() <= 60, (o["color"], got, want)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU rasteriser")
def test_gpu_rasteriser_matches_the_reference():
spec = scenes.spec_batch(list(range(8)), seed=1, px=128)
gpu = render.render_specs(spec, px=128, device="cuda", ss=2)
assert gpu.shape == (8, 3, 128, 128) and float(gpu.min()) >= 0 and float(gpu.max()) <= 1
for i in range(4):
ref = scenes.render_scene(spec["scenes"][i], px=128).astype(np.float32) / 255
mae = float(np.abs(ref - gpu[i].permute(1, 2, 0).cpu().numpy()).mean())
assert mae < 0.02, f"GPU renderer drifted from the reference (MAE {mae:.3f})"
def test_answers_are_grounded():
rng = np.random.default_rng(3)
seen = 0
for _ in range(80):
sc = scenes.make_scene(rng)
q, a = scenes.question_answer(sc, rng)
assert isinstance(q, str) and isinstance(a, str) and a
if q.startswith("how many"):
assert a in scenes.NUM_WORD, a
seen += 1
assert seen > 5, "the generator should produce counting questions"
def test_vocabulary_is_deterministic():
from omni.data import build_vocab
a = build_vocab(seed=5, n=200, cache="/tmp/v1.json")
b = build_vocab(seed=5, n=200, cache="/tmp/v2.json")
assert a == b and len(a) == len(set(a))
from omni.vlm import CombinedVocab
import omni.text as t
v = CombinedVocab(words=a, phones=t.PHONES)
ids = v.words_to_ids(a[:5])
assert v.ids_to_words(ids) == a[:5]
assert len(v) == len(v.words) + len(v.phones)
def test_batch_indices_are_a_pure_function_of_the_step():
"""The resume contract: step N+1 after a restart is the same batch."""
import omni.data as od
first = od.batch_indices_for_step(7, 32, seed=0)
assert first == od.batch_indices_for_step(7, 32, seed=0)
assert first != od.batch_indices_for_step(8, 32, seed=0)
assert sorted(first) == sorted(set(first))
straight = [i for s in range(5) for i in od.batch_indices_for_step(s, 8, 0)]
resumed = [i for s in range(5) for i in od.batch_indices_for_step(s, 8, 0)] # after a "restart"
assert straight == resumed
def test_collate_pads_and_masks_the_answer_region(vocab):
import omni.data as od
spec = scenes.spec_batch(list(range(4)), seed=2, px=64)
batch = od.collate_specs(vocab, spec, device="cpu", px=64)
ids, targets, mask = batch["ids"], batch["targets"], batch["mask"]
assert ids.shape == targets.shape and mask.shape == targets.shape
assert bool(mask.any()) and not bool(mask.all()), "loss covers the answer, not the question"
assert ids.dtype == torch.long and batch["images"].shape[1:] == (3, 64, 64)
for i, start in enumerate(batch["starts"]):
assert not bool(mask[i, :max(0, start - 1)].any())