File size: 3,743 Bytes
e1acc15 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | """AUD1 item 1: production's verifier vs vLLM #61161, on the image's real kernel (CPU interpreter).
Run (Controller, CPU only):
cd offload/audit/verifier_rng
TRITON_INTERPRET=1 CUDA_VISIBLE_DEVICES= python3 -m pytest -q -p no:cacheprovider \
--basetemp=$SCRATCH/pytest test_verifier_rng.py
AUD1_WORKERS (default 16) sets the CPU processes per simulation. All draws are seeded, so the
statistics are reproducible; the chi-square bound has alpha 1e-4 (Wilson-Hilferty).
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent))
import kernel_loader
import multiround as mr
WORKERS = int(os.environ.get("AUD1_WORKERS", "16"))
ALPHA = 1e-4
pytestmark = pytest.mark.skipif(
os.environ.get("TRITON_INTERPRET") != "1" or os.environ.get("CUDA_VISIBLE_DEVICES") != "",
reason="needs TRITON_INTERPRET=1 and CUDA_VISIBLE_DEVICES= (CPU interpreter)",
)
def _check(toy: mr.Toy, res: mr.Result) -> tuple[float, float]:
stat, dof = mr.chi2_joint(toy, res)
return stat, mr.chi2_threshold(dof, ALPHA)
def test_image_kernel_is_pre_61161(tmp_path: Path) -> None:
"""The image's _rejection_kernel keys every acceptance uniform by (request seed, position)."""
src = (kernel_loader.DEFAULT_SRC / "vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py")
text = src.read_text()
assert text.count("u = tl_rand32(seed, pos, includes_zero=False)") == 1
assert "spec_step_seed" not in text
variant = mr.write_pr61161_variant(tmp_path)
assert "spec_step_seed(seed, i) if USE_BLOCK_VERIFICATION else seed" in variant.read_text()
def test_production_standard_one_hot_unbiased_pr_case() -> None:
"""#61161's own deterministic case on production's path: standard, one-hot q, T=1."""
res = mr.simulate(mr.PR_CASE, "standard", 2000, seed=11, workers=WORKERS)
stat, bound = _check(mr.PR_CASE, res)
assert res.accepted > res.trials, "must exercise accepted drafts"
assert res.reread_keys == 0
assert stat < bound, (stat, bound, mr.summary(mr.PR_CASE, res))
def test_production_standard_one_hot_unbiased_context_windows_dead_rows() -> None:
"""Context-dependent target, deterministic drafter, windows 1..5, depth dead rows (-1)."""
res = mr.simulate(mr.PROD_CASE, "standard", 3000, seed=13, workers=WORKERS)
stat, bound = _check(mr.PROD_CASE, res)
assert res.accepted > res.trials // 2
assert res.reread_keys == 0
assert stat < bound, (stat, bound, mr.summary(mr.PROD_CASE, res))
def test_negative_control_block_verification_is_biased_before_fix() -> None:
"""The same kernel in block mode reuses uniforms across rounds and fails the same check.
This is the bug #61161 fixes; it shows the statistic detects it. Production does not use
block verification (rejection_sample_method defaults to "standard"; serve.sh SPEC omits it).
"""
res = mr.simulate(mr.PR_CASE, "block", 2000, seed=11, workers=WORKERS)
stat, bound = _check(mr.PR_CASE, res)
assert res.reread_keys > 0
assert stat > bound, (stat, bound, mr.summary(mr.PR_CASE, res))
def test_positive_control_block_verification_with_61161_change(tmp_path: Path) -> None:
variant = mr.write_pr61161_variant(tmp_path)
res = mr.simulate(mr.PR_CASE, "block", 2000, seed=11, workers=WORKERS, variant=variant)
stat, bound = _check(mr.PR_CASE, res)
assert res.reread_keys == 0
assert stat < bound, (stat, bound, mr.summary(mr.PR_CASE, res))
def test_loader_refuses_gpu_or_compiled_mode(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
with pytest.raises(kernel_loader.LoaderError):
kernel_loader.load()
|