Buckets:
| """CPU-only tests of the newly added API/evaluator transport boundary.""" | |
| import ast | |
| import asyncio | |
| from concurrent.futures import ThreadPoolExecutor | |
| import json | |
| import os | |
| from pathlib import Path | |
| import threading | |
| import time | |
| from types import SimpleNamespace | |
| import urllib.error | |
| import pytest | |
| import common | |
| import remote_judge | |
| def test_exact_config_and_eight_roots(): | |
| import integration | |
| from train_api import config_for | |
| root = Path(os.environ['KERNEL_API3_ROOT']) | |
| integration.configure(root) | |
| cfg = config_for(root) | |
| assert cfg.learning_rate==4e-5 and cfg.kl_penalty_coef==.1 and cfg.num_epochs==3 | |
| assert cfg.phase1_max_tokens==26000 and cfg.local_gpu_context_window==32768 and cfg.lora_rank==32 | |
| assert cfg.adv_estimator=='entropic_adaptive_beta' and cfg.loss_fn=='importance_sampling' | |
| assert not cfg.factorized_action_credit and not cfg.descendant_credit_enabled | |
| ds = asyncio.run(cfg.dataset_builder()) | |
| payload = common.read(root/'control/shared_step0.json') | |
| assert len(ds.sampler._states)==8 and len({s.id for s in ds.sampler._states})==8 | |
| assert all(s.code==payload['best_candidate']['code'] and s.value==payload['best_candidate']['value'] for s in ds.sampler._states) | |
| env = integration.Env(ds.renderer,initial_state=ds.sampler._states[0],sampler=ds.sampler,config=cfg.dataset_builder.config) | |
| from examples.gpu_mode.env import GpuModeEnv | |
| assert env.get_question().startswith(GpuModeEnv.get_question(env)) | |
| assert '## Single-anchor program editing' not in env.get_question() | |
| groups = ds.get_batch(0) | |
| assert len(groups)==8 | |
| for group in groups: | |
| environments = asyncio.run(group.make_envs()) | |
| assert len(environments)==64 | |
| assert len({e._private_slot for e in environments})==64 | |
| assert all(e._private_slot[0]==1 for e in environments) | |
| ds.sampler._current_step=3 | |
| with pytest.raises(RuntimeError,match='fourth'): | |
| ds.get_batch(0) | |
| def test_original_scorer_called_once_on_assembled_program(monkeypatch): | |
| root = Path(os.environ['KERNEL_API3_ROOT']) | |
| import integration | |
| integration.configure(root) | |
| initial = integration.Env.create_initial_state('trimul') | |
| tree = ast.parse(initial.code) | |
| node = next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='custom_kernel') | |
| node.body.insert(0,ast.parse('pfg_cpu_test_marker = 0').body[0]) | |
| patch = '# PFG_ENTRY: custom_kernel\n'+ast.unparse(node) | |
| calls = [] | |
| def evaluate(code,**kwargs): | |
| calls.append((code,kwargs)) | |
| return dict(reward=1.5,correctness=1.,raw_score=1000.,score_us=1000.,status='valid') | |
| from examples.gpu_mode import env as kernel_env | |
| monkeypatch.setattr(kernel_env,'evaluate_submission',evaluate) | |
| result = integration.Env.reward_function(problem_type='trimul',eval_timeout=530).get_reward(patch,initial) | |
| assert result['reward']==1.5 and len(calls)==1 | |
| assert 'custom_kernel' in calls[0][0] and len(calls[0][0])>len(patch) | |
| assert calls[0][1]=={'timeout_seconds':530} | |
| def test_three_step_guard(tmp_path,monkeypatch,step,allowed): | |
| class FakeError(RuntimeError):pass | |
| class Ledger: | |
| def require_paid_authorization(self):pass | |
| transport = SimpleNamespace(Ledger=Ledger,RequestSafetyError=FakeError) | |
| common.install_api_guard(transport,tmp_path) | |
| monkeypatch.setenv('TTT_ALLOW_KERNEL_PRIVATE_API3','1') | |
| ledger = Ledger();ledger.logical_step=step;ledger.lock=threading.Lock() | |
| if allowed: | |
| ledger.require_paid_authorization() | |
| assert common.read(tmp_path/'control/paid_steps'/f'{step:02d}.json')['expected_episodes']==512 | |
| else: | |
| with pytest.raises(FakeError):ledger.require_paid_authorization() | |
| def test_same_request_id_does_not_duplicate_gpu_execution(tmp_path,monkeypatch): | |
| judge = remote_judge.Judge(tmp_path,1) | |
| calls = [] | |
| def evaluate(code,request_id,identity): | |
| calls.append(identity);time.sleep(.03);return {'x':code} | |
| monkeypatch.setattr(judge,'_evaluate',evaluate) | |
| with ThreadPoolExecutor(4) as pool: | |
| values=list(pool.map(lambda _:judge.evaluate('same','a'*32,530),range(4))) | |
| assert values==[{'x':'same'}]*4 and len(calls)==1 | |
| with pytest.raises(remote_judge.InfrastructureError,match='different program'): | |
| judge.evaluate('different','a'*32,530) | |
| def test_execution_budget_cannot_change(tmp_path,timeout): | |
| with pytest.raises(remote_judge.InfrastructureError): | |
| remote_judge.Judge(tmp_path,1).evaluate('code','a'*32,timeout) | |
| def test_real_http_transport_and_authentication(tmp_path,monkeypatch): | |
| (tmp_path/'control').mkdir();(tmp_path/'control/judge_token.txt').write_text('test-only-token') | |
| state=dict(gpu=1,uuid='test-gpu',name='H100',temperature=28,memory_mib=0,pids=[]) | |
| monkeypatch.setattr(remote_judge,'gpu_state',lambda gpu:state) | |
| monkeypatch.setattr(remote_judge.Judge,'evaluate',lambda self,code,rid,timeout:dict( | |
| program_sha256=common.sha(code),image=common.IMAGE,correctness=1.,score_us=123.,reward=1500/123.,raw_score=123.,status='valid')) | |
| original = remote_judge.ThreadingHTTPServer | |
| box=[];ready=threading.Event() | |
| def server(*a,**k): | |
| value=original(*a,**k);box.append(value);ready.set();return value | |
| monkeypatch.setattr(remote_judge,'ThreadingHTTPServer',server) | |
| thread=threading.Thread(target=remote_judge.serve,args=(tmp_path,'127.0.0.1',0,1,'127.0.0.1'),daemon=True) | |
| thread.start();assert ready.wait(5) | |
| url='http://127.0.0.1:'+str(box[0].server_address[1]) | |
| try: | |
| client=remote_judge.Client(url,'test-only-token') | |
| assert client.health()['state']==state | |
| assert client.evaluate('complete program')['score_us']==123. | |
| with pytest.raises(urllib.error.HTTPError) as error: | |
| remote_judge.Client(url,'wrong-token').health() | |
| assert error.value.code==403 | |
| finally: | |
| box[0].shutdown();box[0].server_close();thread.join(5) | |
| def test_mismatched_program_result_rejected(monkeypatch): | |
| client=remote_judge.Client('http://127.0.0.1:1','unused') | |
| monkeypatch.setattr(client,'_request',lambda *a,**k:dict(program_sha256='wrong',image=common.IMAGE)) | |
| with pytest.raises(remote_judge.InfrastructureError,match='mismatched'): | |
| client.evaluate('program') | |
| def test_docker_manifest_and_config_identity(monkeypatch,canonical,allowed): | |
| client=remote_judge.Client('http://127.0.0.1:1','unused') | |
| monkeypatch.setattr(client,'_request',lambda *a,**k:dict(image=common.IMAGE_MANIFEST, | |
| canonical_image_config_sha256=canonical,execution_seconds=530)) | |
| if allowed: | |
| assert client.health()['image']==common.IMAGE_MANIFEST | |
| else: | |
| with pytest.raises(remote_judge.InfrastructureError,match='protocol mismatch'): | |
| client.health() | |
Xet Storage Details
- Size:
- 7.01 kB
- Xet hash:
- 7c297edec16913eb0cb4307ab38c7dfd6905ef809d5e81e459d7de672a3d6ac0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.