zefancainlp's picture
download
raw
7.84 kB
"""Real native environment integration, with the GPU evaluator mocked on CPU."""
import argparse
import hashlib
import json
from pathlib import Path
import pytest
from examples.gpu_mode.env import GpuModeEnv, SingleAnchorTriMulEnv
from examples.gpu_mode.official import original_question
from examples.gpu_mode.private_entry import VERSION, merge_private_entry
from examples.gpu_mode.private_entry_env import (
PrivateEntryTriMulEnv, PrivateEntryTriMulState, PrivateEntryTriMulRewardEvaluator,
)
from ttt_discover.tinker_utils.dataset_builder import VerifyResult
from ttt_discover.tinker_utils.state import State
from ttt_discover.runtime.shared_step0 import export_step0
from ttt_discover.runtime.launch import resolve
from ttt_discover.runtime.train import build_config, environment_for
from ttt_discover.runtime.topology import FOUR_GPU_PROFILE
ROOT = '''import triton
import triton.language as tl
@triton.jit
def kernel(X, N: tl.constexpr, BLOCK: tl.constexpr):
return
def launch(data):
kernel[(1,)](data, 1, BLOCK=32)
return data
def other(data):
return launch(data)
def custom_kernel(data):
return launch(data)
def unused(data):
return data
'''
PATCH = '''# PFG_ENTRY: launch
@triton.jit
def kernel(X, N: tl.constexpr, BLOCK: tl.constexpr):
value = N + 1
'''
def state():
return PrivateEntryTriMulState.from_root({"id":"test-only-root", "code":ROOT, "value":-1500.0})
def env():
obj = PrivateEntryTriMulEnv.__new__(PrivateEntryTriMulEnv)
obj.problem_type = "trimul"
obj.initial_state = state()
return obj
def test_official_prompt_prefix_unchanged_and_old_action_prompt_removed():
e = env()
prompt = e.get_question()
assert prompt.startswith(original_question()(e))
assert "## Single-anchor program editing" not in prompt
assert "# PFG_ENTRY:" in prompt
assert "- kernel(" in prompt and "Triton JIT" in prompt
assert "- unused(" not in prompt
assert "Stored but currently unreachable functions" in prompt
def test_state_preserves_step0_and_roundtrip():
parent = state()
assert parent.step0_source == ROOT
assert PrivateEntryTriMulState.from_dict(parent.to_dict()).to_dict() == parent.to_dict()
with pytest.raises(ValueError, match="version mismatch"):
PrivateEntryTriMulState(private_entry_version="wrong")
def test_reward_calls_original_evaluator_once(monkeypatch):
import examples.gpu_mode.env as old
calls = []
def evaluate(source, *, timeout_seconds):
calls.append((source, timeout_seconds))
return {"reward":1.25, "correctness":1.0, "raw_score":1200.0,
"score_us":1200.0, "status":"valid"}
monkeypatch.setattr(old, "evaluate_submission", evaluate)
evaluator = PrivateEntryTriMulRewardEvaluator(problem_type="trimul", eval_timeout=530)
result = evaluator.get_reward(PATCH, state())
assert result["reward"] == 1.25 and result["raw_score"] == 1200.0
assert result["correctness"] == 1.0
assert calls == [(merge_private_entry(ROOT, PATCH).source, 530)]
def test_static_error_never_calls_gpu_evaluator(monkeypatch):
import examples.gpu_mode.env as old
monkeypatch.setattr(old, "evaluate_submission", lambda *a, **k: pytest.fail("GPU call on invalid transaction"))
result = PrivateEntryTriMulRewardEvaluator(problem_type="trimul").get_reward(
"def custom_kernel(data):\n return nonexistent(data)\n", state())
assert result["reward"] == result["correctness"] == 0
assert "Undefined global" in result["msg"]
def test_child_state_sign_patch_history_and_private_audit():
e = env()
outs = VerifyResult(1.25, "mock official evaluator", 1.0, 1200.0, [], "")
child = e._create_next_state(2, PATCH, outs)
assert child.value == -1200.0 and child.timestep == 2
assert child.patch_history == [PATCH]
assert child.step0_source == ROOT
assert child.entry_transactions[-1]["entry"] == "launch"
assert child.entry_transactions[-1]["edited_symbols"] == ["kernel"]
assert child.entry_transactions[-1]["private_symbols"]
assert PrivateEntryTriMulState.from_dict(child.to_dict()).to_dict() == child.to_dict()
def test_format_error_is_available_for_diagnosis():
e = env()
assert not e.check_format("def custom_kernel(data):\n return nonexistent(data)\n")
assert "Undefined global" in e._private_error
assert e.check_format(PATCH)
assert e._private_error is None
def test_metrics_do_not_call_old_single_anchor_merger(monkeypatch):
import examples.gpu_mode.env as old
monkeypatch.setattr(old, "merge_single_anchor", lambda *a, **k: pytest.fail("old grammar used for private transaction"))
e = env()
outs = VerifyResult(1.25, "mock official evaluator", 1.0, 1200.0, [], "")
metrics = e._build_metrics(outs, True, {"content":PATCH}, PATCH)
assert metrics["single_anchor_pfg/variant"] == "trimul_private_entry"
assert metrics["private_entry/old_bindings_replaced"] == 1
assert metrics["private_entry/edited_symbols"] == ["kernel"]
def test_raw_transaction_and_lowered_wiring_are_both_logged(monkeypatch, tmp_path):
monkeypatch.setattr(SingleAnchorTriMulEnv, "_record_function_analysis", lambda *a, **k: None)
e = env()
e.log_path = str(tmp_path)
outs = VerifyResult(1.25, "mock official evaluator", 1.0, 1200.0, [], "")
child = e._create_next_state(0, PATCH, outs)
response = "preserved model reasoning\n" + PATCH
e._record_function_analysis(0, {"content":response}, PATCH, outs, {}, True, True, child)
records = list(tmp_path.rglob("*.json"))
assert len(records) == 1
row = json.loads(records[0].read_text())
assert row["response"] == response and row["model_patch"] == PATCH
assert row["lowered_patch"] == merge_private_entry(ROOT, PATCH).lowered_patch
assert row["logical_update"] == 1 and row["step"] == 0
def options(tmp_path, method="single_anchor_private"):
row = State(id="test-only-seed", timestep=0, code=ROOT, value=-1500.0, construction=[]).to_dict()
sampler = tmp_path/"sampler.json"
sampler.write_text(json.dumps({"states":[row], "initial_states":[], "puct_T":1, "puct_n":{}, "puct_m":{}}))
step0 = tmp_path/"step0.json"
export_step0("trimul", sampler, step0)
return argparse.Namespace(task="trimul", method=method, model_path=tmp_path/"model",
run_dir=tmp_path/"new-run", resume=False, smoke=False, updates=3,
rollout_gpus=None, execution_profile=FOUR_GPU_PROFILE, shared_step0=step0)
def test_native_dispatch_reuses_shared_step0_and_original_rl(tmp_path):
c = resolve(options(tmp_path))
assert c["method"] == "single_anchor_private"
assert c["kernel_private_entry_spec"]["version"] == VERSION
cls = environment_for(c)
assert issubclass(cls, PrivateEntryTriMulEnv)
initial = cls.create_initial_state("trimul")
assert isinstance(initial, PrivateEntryTriMulState)
assert initial.code == ROOT and initial.value == -1500.0
cfg = build_config(c)
assert (cfg.group_size, cfg.groups_per_batch, cfg.num_epochs) == (64, 8, 3)
assert cfg.parent_selection_policy == "puct"
assert cfg.adv_estimator == "entropic_adaptive_beta"
assert cfg.learning_rate == c["learning_rate"]
assert cfg.kl_penalty_coef == c["kl_penalty_coef"]
def test_historical_single_anchor_dispatch_is_unchanged(tmp_path):
c = resolve(options(tmp_path, method="single_anchor"))
assert issubclass(environment_for(c), SingleAnchorTriMulEnv)
assert not issubclass(environment_for(c), PrivateEntryTriMulEnv)
assert "kernel_private_entry_spec" not in c
def test_runtime_rejects_compiler_drift(tmp_path):
c = resolve(options(tmp_path))
c["kernel_private_entry_spec"]["compiler_sha256"] = "changed"
with pytest.raises(ValueError, match="resolved configuration"):
environment_for(c)

Xet Storage Details

Size:
7.84 kB
·
Xet hash:
2815e4d6e076a888170f01ad09fa6fed188ea6a57cafe5a596e3b26acd0b2e4a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.