zefancainlp's picture
download
raw
7.01 kB
"""CPU-only tests of the newly added API/evaluator transport boundary."""
import ast
import asyncio
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import threading
import time
from types import SimpleNamespace
import urllib.error
import pytest
import common
import remote_judge
def test_exact_config_and_eight_roots():
import integration
from train_api import config_for
root = Path(os.environ['KERNEL_API3_ROOT'])
integration.configure(root)
cfg = config_for(root)
assert cfg.learning_rate==4e-5 and cfg.kl_penalty_coef==.1 and cfg.num_epochs==3
assert cfg.phase1_max_tokens==26000 and cfg.local_gpu_context_window==32768 and cfg.lora_rank==32
assert cfg.adv_estimator=='entropic_adaptive_beta' and cfg.loss_fn=='importance_sampling'
assert not cfg.factorized_action_credit and not cfg.descendant_credit_enabled
ds = asyncio.run(cfg.dataset_builder())
payload = common.read(root/'control/shared_step0.json')
assert len(ds.sampler._states)==8 and len({s.id for s in ds.sampler._states})==8
assert all(s.code==payload['best_candidate']['code'] and s.value==payload['best_candidate']['value'] for s in ds.sampler._states)
env = integration.Env(ds.renderer,initial_state=ds.sampler._states[0],sampler=ds.sampler,config=cfg.dataset_builder.config)
from examples.gpu_mode.env import GpuModeEnv
assert env.get_question().startswith(GpuModeEnv.get_question(env))
assert '## Single-anchor program editing' not in env.get_question()
groups = ds.get_batch(0)
assert len(groups)==8
for group in groups:
environments = asyncio.run(group.make_envs())
assert len(environments)==64
assert len({e._private_slot for e in environments})==64
assert all(e._private_slot[0]==1 for e in environments)
ds.sampler._current_step=3
with pytest.raises(RuntimeError,match='fourth'):
ds.get_batch(0)
def test_original_scorer_called_once_on_assembled_program(monkeypatch):
root = Path(os.environ['KERNEL_API3_ROOT'])
import integration
integration.configure(root)
initial = integration.Env.create_initial_state('trimul')
tree = ast.parse(initial.code)
node = next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='custom_kernel')
node.body.insert(0,ast.parse('pfg_cpu_test_marker = 0').body[0])
patch = '# PFG_ENTRY: custom_kernel\n'+ast.unparse(node)
calls = []
def evaluate(code,**kwargs):
calls.append((code,kwargs))
return dict(reward=1.5,correctness=1.,raw_score=1000.,score_us=1000.,status='valid')
from examples.gpu_mode import env as kernel_env
monkeypatch.setattr(kernel_env,'evaluate_submission',evaluate)
result = integration.Env.reward_function(problem_type='trimul',eval_timeout=530).get_reward(patch,initial)
assert result['reward']==1.5 and len(calls)==1
assert 'custom_kernel' in calls[0][0] and len(calls[0][0])>len(patch)
assert calls[0][1]=={'timeout_seconds':530}
@pytest.mark.parametrize('step,allowed',[(0,False),(1,True),(3,True),(4,False)])
def test_three_step_guard(tmp_path,monkeypatch,step,allowed):
class FakeError(RuntimeError):pass
class Ledger:
def require_paid_authorization(self):pass
transport = SimpleNamespace(Ledger=Ledger,RequestSafetyError=FakeError)
common.install_api_guard(transport,tmp_path)
monkeypatch.setenv('TTT_ALLOW_KERNEL_PRIVATE_API3','1')
ledger = Ledger();ledger.logical_step=step;ledger.lock=threading.Lock()
if allowed:
ledger.require_paid_authorization()
assert common.read(tmp_path/'control/paid_steps'/f'{step:02d}.json')['expected_episodes']==512
else:
with pytest.raises(FakeError):ledger.require_paid_authorization()
def test_same_request_id_does_not_duplicate_gpu_execution(tmp_path,monkeypatch):
judge = remote_judge.Judge(tmp_path,1)
calls = []
def evaluate(code,request_id,identity):
calls.append(identity);time.sleep(.03);return {'x':code}
monkeypatch.setattr(judge,'_evaluate',evaluate)
with ThreadPoolExecutor(4) as pool:
values=list(pool.map(lambda _:judge.evaluate('same','a'*32,530),range(4)))
assert values==[{'x':'same'}]*4 and len(calls)==1
with pytest.raises(remote_judge.InfrastructureError,match='different program'):
judge.evaluate('different','a'*32,530)
@pytest.mark.parametrize('timeout',[300,529,531])
def test_execution_budget_cannot_change(tmp_path,timeout):
with pytest.raises(remote_judge.InfrastructureError):
remote_judge.Judge(tmp_path,1).evaluate('code','a'*32,timeout)
def test_real_http_transport_and_authentication(tmp_path,monkeypatch):
(tmp_path/'control').mkdir();(tmp_path/'control/judge_token.txt').write_text('test-only-token')
state=dict(gpu=1,uuid='test-gpu',name='H100',temperature=28,memory_mib=0,pids=[])
monkeypatch.setattr(remote_judge,'gpu_state',lambda gpu:state)
monkeypatch.setattr(remote_judge.Judge,'evaluate',lambda self,code,rid,timeout:dict(
program_sha256=common.sha(code),image=common.IMAGE,correctness=1.,score_us=123.,reward=1500/123.,raw_score=123.,status='valid'))
original = remote_judge.ThreadingHTTPServer
box=[];ready=threading.Event()
def server(*a,**k):
value=original(*a,**k);box.append(value);ready.set();return value
monkeypatch.setattr(remote_judge,'ThreadingHTTPServer',server)
thread=threading.Thread(target=remote_judge.serve,args=(tmp_path,'127.0.0.1',0,1,'127.0.0.1'),daemon=True)
thread.start();assert ready.wait(5)
url='http://127.0.0.1:'+str(box[0].server_address[1])
try:
client=remote_judge.Client(url,'test-only-token')
assert client.health()['state']==state
assert client.evaluate('complete program')['score_us']==123.
with pytest.raises(urllib.error.HTTPError) as error:
remote_judge.Client(url,'wrong-token').health()
assert error.value.code==403
finally:
box[0].shutdown();box[0].server_close();thread.join(5)
def test_mismatched_program_result_rejected(monkeypatch):
client=remote_judge.Client('http://127.0.0.1:1','unused')
monkeypatch.setattr(client,'_request',lambda *a,**k:dict(program_sha256='wrong',image=common.IMAGE))
with pytest.raises(remote_judge.InfrastructureError,match='mismatched'):
client.evaluate('program')
@pytest.mark.parametrize('canonical,allowed',[(common.IMAGE,True),('wrong-image',False)])
def test_docker_manifest_and_config_identity(monkeypatch,canonical,allowed):
client=remote_judge.Client('http://127.0.0.1:1','unused')
monkeypatch.setattr(client,'_request',lambda *a,**k:dict(image=common.IMAGE_MANIFEST,
canonical_image_config_sha256=canonical,execution_seconds=530))
if allowed:
assert client.health()['image']==common.IMAGE_MANIFEST
else:
with pytest.raises(remote_judge.InfrastructureError,match='protocol mismatch'):
client.health()

Xet Storage Details

Size:
7.01 kB
·
Xet hash:
7c297edec16913eb0cb4307ab38c7dfd6905ef809d5e81e459d7de672a3d6ac0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.