Buckets:
| """No-network tests. All clients in these tests are local fakes.""" | |
| from __future__ import annotations | |
| import ast | |
| import asyncio | |
| import copy | |
| from dataclasses import dataclass | |
| import json | |
| import os | |
| from pathlib import Path | |
| import tempfile | |
| from types import SimpleNamespace | |
| import unittest | |
| from unittest.mock import patch | |
| import tinker | |
| import torch | |
| from api_transport import Ledger, ROLLOUT_SLOT, SamplingProxy, RequestSafetyError, atomic, hash_input, sample_payload, cached_sample | |
| from component_core import (build_candidates, gate_layout, metadata_specialize, observed_projection_plans, | |
| profile_contract, sha, structural_key, transfer_projection_plan, verify_uniform_mask_default, | |
| component_fingerprint, normalized_consumer, helpers) | |
| def observed(prefix=5., suffix=5., layout='BNNH4'): | |
| cases = [] | |
| for index, n in enumerate([256, 768, 256, 512, 1024, 768, 1024]): | |
| shape = dict(BHNN3=[128, n, n], BHNN4=[1, 128, n, n], BNNH4=[1, n, n, 128], MH2=[n * n, 128])[layout] | |
| cases.append(dict(B=1, N=n, H=128, mid=dict(shape=[128, n, n], dtype='torch.float16'), | |
| gate=dict(shape=shape, dtype='torch.float16'), prefix_us=[prefix] * 3, suffix_us=[suffix] * 3)) | |
| return dict(complete=True, cases=cases) | |
| SOURCE = '''import torch | |
| def change_front(x): | |
| return x + 0 | |
| def change_tail(x): | |
| return x - 0 | |
| def batched_matmul_triton(a,b): | |
| return torch.bmm(a,b.transpose(1,2)) | |
| def custom_kernel(data): | |
| inp, mask, weights, cfg = data | |
| dim = cfg['dim'] | |
| hidden_dim = cfg['hidden_dim'] | |
| B, N, _, _ = inp.shape | |
| device = inp.device | |
| out_gate = inp[..., :hidden_dim] | |
| x = change_front(inp).reshape(B*hidden_dim,N,N) | |
| hidden = batched_matmul_triton(x,x) | |
| return change_tail(hidden).reshape(B,N,N,hidden_dim) * out_gate | |
| ''' | |
| class CoreTests(unittest.TestCase): | |
| def test_all_four_layouts(self): | |
| for layout in ['BHNN3', 'BHNN4', 'BNNH4', 'MH2']: | |
| self.assertEqual(profile_contract(observed(layout=layout)), layout) | |
| def test_ambiguous_layout_not_guessed(self): | |
| with self.assertRaises(ValueError): | |
| gate_layout([1, 128, 128, 128], 1, 128, 128) | |
| def test_incomplete_profile_rejected(self): | |
| p = observed() | |
| p['cases'].pop() | |
| with self.assertRaises(ValueError): | |
| profile_contract(p) | |
| def test_dtype_rejected(self): | |
| p = observed() | |
| p['cases'][0]['mid']['dtype'] = 'torch.float32' | |
| with self.assertRaises(ValueError): | |
| profile_contract(p) | |
| def test_changed_layout_policy_rejected(self): | |
| p = observed() | |
| p['cases'][0]['gate']['shape'] = [1, 128, 256, 256] | |
| with self.assertRaises(ValueError): | |
| profile_contract(p) | |
| def test_negative_component_time_rejected(self): | |
| p = observed() | |
| p['cases'][0]['prefix_us'][0] = -1 | |
| with self.assertRaises(ValueError): | |
| profile_contract(p) | |
| def test_mid_layout_rejected(self): | |
| p = observed() | |
| p['cases'][0]['mid']['shape'] = [1, 256, 256, 128] | |
| with self.assertRaises(ValueError): | |
| profile_contract(p) | |
| def test_mask_correction_is_only_default(self): | |
| original = "def custom_kernel(data):\n x = data[3].get('nomask', True)\n return x\n" | |
| fixed, n = verify_uniform_mask_default(original) | |
| self.assertEqual(n, 1) | |
| self.assertIn("get('nomask', False)", fixed) | |
| again, n = verify_uniform_mask_default(fixed) | |
| self.assertEqual(n, 0) | |
| self.assertEqual(again, fixed) | |
| def test_joint_search_keeps_individually_worse_candidates(self): | |
| parent = dict(id='p', code=SOURCE, score_us=100., profile=observed()) | |
| a = dict(id='a', code=SOURCE.replace('x + 0', 'x + 1'), score_us=150., profile=observed(prefix=1.)) | |
| b = dict(id='b', code=SOURCE.replace('x - 0', 'x - 1'), score_us=160., profile=observed(suffix=1.)) | |
| programs, audit = build_candidates(parent, [a, b], per_side=2) | |
| self.assertTrue(any(p['front_id'] == 'a' and p['tail_id'] == 'b' for p in programs)) | |
| self.assertLessEqual(len(programs), 9) | |
| self.assertEqual(a['score_us'], 150.) | |
| for p in programs: | |
| compile(p['code'], '<joint>', 'exec') | |
| def test_no_borrowed_components_when_only_one_source(self): | |
| parent = dict(id='p', code=SOURCE, score_us=100., profile=observed()) | |
| programs, audit = build_candidates(parent, [], per_side=6) | |
| self.assertEqual(programs, []) | |
| self.assertEqual(audit['input_programs'], 1) | |
| def test_no_original_mutation(self): | |
| parent = dict(id='p', code=SOURCE, score_us=100., profile=observed()) | |
| children = [dict(id='a', code=SOURCE.replace('x + 0', 'x + 1'), score_us=90., profile=observed())] | |
| before = copy.deepcopy((parent, children)) | |
| build_candidates(parent, children) | |
| self.assertEqual(before, (parent, children)) | |
| def test_no_numeric_changes_from_scalar_specialization(self): | |
| source = '''import triton | |
| import triton.language as tl | |
| @triton.jit | |
| def k(C, N): | |
| tl.store(C + tl.arange(0,16),N) | |
| def custom_kernel(data): | |
| x = data[0] | |
| B,N,D,H = x.shape | |
| C = x | |
| k[(1,)](C,N) | |
| return C | |
| ''' | |
| fixed, edits = metadata_specialize(source) | |
| tree = ast.parse(fixed) | |
| k = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'k') | |
| self.assertIsNone(k.args.args[0].annotation) | |
| self.assertEqual(ast.unparse(k.args.args[1].annotation), 'tl.constexpr') | |
| before = next(n for n in ast.parse(source).body if isinstance(n, ast.FunctionDef) and n.name == 'k') | |
| self.assertEqual(ast.dump(ast.Module(body=before.body, type_ignores=[])), ast.dump(ast.Module(body=k.body, type_ignores=[]))) | |
| def test_rebound_scalar_cannot_be_marked_constant(self): | |
| source = '''import triton | |
| import triton.language as tl | |
| @triton.jit | |
| def k(C): | |
| tl.store(C,0) | |
| def custom_kernel(data): | |
| C = 3 | |
| C = data[0] | |
| k[(1,)](C) | |
| return C | |
| ''' | |
| fixed, edits = metadata_specialize(source) | |
| self.assertEqual(edits, []) | |
| def test_docstring_only_dedup(self): | |
| self.assertEqual(structural_key(SOURCE), structural_key('"a comment"\n' + SOURCE)) | |
| def test_generated_namespaces_are_not_new_components(self): | |
| tools, _, _ = helpers() | |
| tree, names = tools.namespaced(ast.parse(SOURCE), 'component_front_') | |
| next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == names['custom_kernel']).name = 'custom_kernel' | |
| # The cut recognizer deliberately supports only its two proven BMM | |
| # entry names. Test namespace invariance within that admitted ABI, | |
| # rather than accidentally constructing an unsupported contraction. | |
| for node in ast.walk(tree): | |
| if isinstance(node, ast.FunctionDef) and node.name == names['batched_matmul_triton']: | |
| node.name = 'batched_matmul_triton' | |
| if isinstance(node, ast.Name) and node.id == names['batched_matmul_triton']: | |
| node.id = 'batched_matmul_triton' | |
| namespaced = ast.unparse(ast.fix_missing_locations(tree)) | |
| for side in ['front', 'tail']: | |
| self.assertEqual(component_fingerprint(SOURCE, tools, side), component_fingerprint(namespaced, tools, side)) | |
| def test_referenced_global_constants_are_not_erased_by_dedup(self): | |
| tools, _, _ = helpers() | |
| source = 'MY_GLOBAL = 1\n' + SOURCE.replace('x + 0', 'x + MY_GLOBAL') | |
| other = source.replace('MY_GLOBAL = 1', 'MY_GLOBAL = 2') | |
| self.assertNotEqual(component_fingerprint(source, tools, 'front'), component_fingerprint(other, tools, 'front')) | |
| def test_own_old_gate_adapter_is_peeled_before_recombination(self): | |
| tools, _, _ = helpers() | |
| source = SOURCE.replace('return change_tail(hidden)', | |
| '_component_consumer_out_gate = out_gate.permute(0,3,1,2).contiguous()\n return change_tail(hidden)') | |
| source = source.replace('* out_gate\n', '* _component_consumer_out_gate\n') | |
| profile = observed() | |
| for case in profile['cases']: | |
| case['gate_aliases'] = {'_component_consumer_out_gate': {'shape': [1, 128, case['N'], case['N']], 'dtype': 'torch.float16'}} | |
| template, layout, audit = normalized_consumer(source, profile, tools) | |
| self.assertEqual(layout, 'BHNN4') | |
| self.assertEqual(audit['peeled_gate_adapters'], 1) | |
| self.assertNotIn('_component_consumer_out_gate', template) | |
| self.assertIn('* out_gate', template) | |
| class FakeSampling: | |
| def __init__(self): | |
| self.calls = 0 | |
| async def sample_async(self, **kwargs): | |
| self.calls += 1 | |
| return cached_sample(dict(sequences=[dict(tokens=[11, 12], logprobs=[-.1, -.2], stop_reason='stop')])) | |
| async def compute_logprobs_async(self, model_input, **kwargs): | |
| self.calls += 1 | |
| return [None, -.2] | |
| class CacheTests(unittest.IsolatedAsyncioTestCase): | |
| def setUp(self): | |
| self.tmp = tempfile.TemporaryDirectory() | |
| self.ledger = Ledger(self.tmp.name, enabled=True) | |
| self.ledger.logical_step = 1 | |
| self.fake = FakeSampling() | |
| self.proxy = SamplingProxy(self.fake, self.ledger, policy='boundary-0') | |
| self.prompt = tinker.ModelInput.from_ints([1, 2]) | |
| self.params = tinker.SamplingParams(max_tokens=8, temperature=1.) | |
| self.slot = ROLLOUT_SLOT.set((1, 0, 0)) | |
| self.env = patch.dict(os.environ, {'TTT_ALLOW_PAID_KERNEL_API10': '1'}) | |
| self.env.start() | |
| def tearDown(self): | |
| ROLLOUT_SLOT.reset(self.slot) | |
| self.env.stop() | |
| self.tmp.cleanup() | |
| async def test_exact_response_reused_without_second_model_call(self): | |
| first = await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| resumed = SamplingProxy(self.fake, self.ledger, policy='boundary-0') | |
| second = await resumed.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| self.assertEqual(sample_payload(first), sample_payload(second)) | |
| self.assertEqual(self.fake.calls, 1) | |
| async def test_third_phase_refused(self): | |
| for _ in range(2): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| with self.assertRaises(RequestSafetyError): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| self.assertEqual(self.fake.calls, 2) | |
| async def test_disabled_paid_gate(self): | |
| self.ledger.enabled = False | |
| with self.assertRaises(RequestSafetyError): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| self.assertEqual(self.fake.calls, 0) | |
| async def test_no_untracked_generation(self): | |
| ROLLOUT_SLOT.set(None) | |
| with self.assertRaises(RequestSafetyError): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| async def test_no_more_than_64_per_group(self): | |
| ROLLOUT_SLOT.set((1, 0, 64)) | |
| with self.assertRaises(RequestSafetyError): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| async def test_no_more_than_8_groups(self): | |
| ROLLOUT_SLOT.set((1, 8, 0)) | |
| with self.assertRaises(RequestSafetyError): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| async def test_unknown_paid_request_not_retried(self): | |
| identity = dict(slot=[1, 0, 0], phase=1, policy='boundary-0', prompt_sha256=hash_input(self.prompt), | |
| settings=self.params.model_dump(mode='json'), kwargs={}) | |
| _, pending = self.ledger.cache_paths(json.dumps(identity, sort_keys=True)) | |
| atomic(pending, dict(identity=identity)) | |
| with self.assertRaises(RequestSafetyError): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| self.assertEqual(self.fake.calls, 0) | |
| async def test_reference_not_a_generator(self): | |
| proxy = SamplingProxy(self.fake, self.ledger, policy='base', reference=True) | |
| with self.assertRaises(RequestSafetyError): | |
| await proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| async def test_reference_cache_reused(self): | |
| proxy = SamplingProxy(self.fake, self.ledger, policy='base', reference=True) | |
| a = await proxy.compute_logprobs_async(self.prompt) | |
| b = await proxy.compute_logprobs_async(self.prompt) | |
| self.assertEqual(a, b) | |
| self.assertEqual(self.fake.calls, 1) | |
| async def test_policy_boundary_is_part_of_identity(self): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| changed = SamplingProxy(self.fake, self.ledger, policy='boundary-1') | |
| with self.assertRaises(RequestSafetyError): | |
| await changed.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| self.assertEqual(self.fake.calls, 1) | |
| async def test_same_slot_different_prompt_cannot_be_resampled(self): | |
| await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params) | |
| changed = SamplingProxy(self.fake, self.ledger, policy='boundary-0') | |
| with self.assertRaises(RequestSafetyError): | |
| await changed.sample_async(prompt=tinker.ModelInput.from_ints([3, 4]), num_samples=1, sampling_params=self.params) | |
| self.assertEqual(self.fake.calls, 1) | |
| class IntegrationTests(unittest.IsolatedAsyncioTestCase): | |
| async def test_native_512_sample_update_uses_no_synthetic_program_tokens(self): | |
| from api_transport import TrainingProxy | |
| from ttt_discover.rl.types import Trajectory, TrajectoryGroup, Transition | |
| from ttt_discover.tinker_utils.completers import TokensWithLogprobs | |
| from ttt_discover.rl.data_processing import assemble_training_data | |
| from ttt_discover.rl.advantages import compute_group_advantages | |
| from ttt_discover.rl.train import train_step, incorporate_kl_penalty | |
| groups, advantages = [], [] | |
| for group in range(8): | |
| trajectories = [] | |
| rewards = [] | |
| for i in range(64): | |
| action = [10 + group, 100 + i] | |
| transition = Transition(ob=tinker.ModelInput.from_ints([1, 2, 3]), | |
| ac=TokensWithLogprobs(tokens=action, maybe_logprobs=[-.2, -.3], maybe_mask=[1., 1.]), | |
| reward=i / 64., episode_done=True, metrics={}) | |
| trajectories.append(Trajectory(transitions=[transition], final_ob=tinker.ModelInput.empty())) | |
| rewards.append(i / 64.) | |
| groups.append(TrajectoryGroup(trajectories_G=trajectories, final_rewards_G=[0.] * 64, metrics_G=[{} for _ in range(64)])) | |
| advantages.append(compute_group_advantages(rewards, 'entropic_adaptive_beta')) | |
| data, metadata = assemble_training_data(groups, advantages) | |
| self.assertEqual(len(data), 512) | |
| self.assertEqual(len(metadata), 512) | |
| self.assertEqual(sum(float(d.loss_fn_inputs['mask'].to_torch().sum()) for d in data), 1024.) | |
| captured = [] | |
| class Future: | |
| def __init__(self, result): self.value = result | |
| async def result_async(self): return self.value | |
| class FakeTraining: | |
| async def forward_backward_async(self, rows, **kwargs): | |
| captured.append(('fwb', rows, kwargs)) | |
| self.rows = rows | |
| return Future(SimpleNamespace(loss_fn_outputs=[{'logprobs': tinker.TensorData.from_torch(torch.zeros(d.model_input.length))} for d in rows])) | |
| async def optim_step_async(self, params): | |
| captured.append(('adam', params)) | |
| return Future(SimpleNamespace()) | |
| class Reference: | |
| async def compute_logprobs_async(self, model_input): | |
| return [None] + [-.1] * (model_input.length - 1) | |
| with tempfile.TemporaryDirectory() as tmp, patch.dict(os.environ, {'TTT_ALLOW_PAID_KERNEL_API10': '1'}): | |
| ledger = Ledger(tmp, enabled=True) | |
| ledger.logical_step = 1 | |
| training = TrainingProxy(FakeTraining(), SimpleNamespace(ledger=ledger)) | |
| original_targets = [d.loss_fn_inputs['target_tokens'].to_torch().clone() for d in data] | |
| await incorporate_kl_penalty(data, Reference(), .1) | |
| trained = await train_step(data, training, learning_rate=4e-5, num_substeps=1, loss_fn='importance_sampling') | |
| self.assertEqual(len(trained), 512) | |
| self.assertEqual([r[0] for r in captured], ['fwb', 'adam']) | |
| self.assertEqual(len(captured[0][1]), 512) | |
| self.assertTrue(all('mask' not in d.loss_fn_inputs for d in captured[0][1])) | |
| self.assertTrue(all(torch.equal(before, after.loss_fn_inputs['target_tokens'].to_torch()) for before, after in zip(original_targets, data))) | |
| async def test_64_original_trajectories_survive_search_unchanged(self): | |
| import integration | |
| called = [] | |
| runtime = SimpleNamespace(fuse_group=lambda *args: called.append(args)) | |
| trajectories = [SimpleNamespace(transitions=[SimpleNamespace(ac=SimpleNamespace(tokens=[i, i + 1]), reward=i / 64.)]) for i in range(64)] | |
| envs = [SimpleNamespace(sampler='sampler') for _ in range(64)] | |
| builder = integration.ComponentGroupBuilder(lambda: None, 64, initial_state='parent', group_slot=0, logical_step=1) | |
| with patch.object(integration, 'RUNTIME', runtime): | |
| rewards = await builder.compute_group_rewards(trajectories, envs) | |
| self.assertEqual(rewards, [(0., {})] * 64) | |
| self.assertEqual(len(called), 1) | |
| self.assertEqual([t.transitions[0].reward for t in trajectories], [i / 64. for i in range(64)]) | |
| async def test_fusion_admission_is_on_the_native_event_loop(self): | |
| import integration | |
| import threading | |
| loop_thread = threading.get_ident() | |
| node = SimpleNamespace(code='synthetic search node, not a trajectory') | |
| updates = [] | |
| sampler = SimpleNamespace(update_states=lambda *args, **kwargs: updates.append((threading.get_ident(), args))) | |
| def search(*args): | |
| self.assertNotEqual(threading.get_ident(), loop_thread) | |
| return node | |
| runtime = SimpleNamespace(fuse_group=search) | |
| trajectories = [SimpleNamespace(transitions=[SimpleNamespace(ac=SimpleNamespace(tokens=[i]), reward=float(i))]) for i in range(64)] | |
| envs = [SimpleNamespace(sampler=sampler) for _ in range(64)] | |
| builder = integration.ComponentGroupBuilder(lambda: None, 64, initial_state='parent', group_slot=0, logical_step=1) | |
| with patch.object(integration, 'RUNTIME', runtime): | |
| rewards = await builder.compute_group_rewards(trajectories, envs) | |
| self.assertEqual(len(updates), 1) | |
| self.assertEqual(updates[0][0], loop_thread) | |
| self.assertIs(updates[0][1][0][0], node) | |
| self.assertEqual(rewards, [(0., {})] * 64) | |
| async def test_every_episode_has_a_distinct_stable_slot(self): | |
| import integration | |
| builder = integration.ComponentGroupBuilder(lambda: SimpleNamespace(), 64, initial_state='parent', group_slot=7, logical_step=9) | |
| envs = await builder.make_envs() | |
| self.assertEqual({e._component_slot for e in envs}, {(9, 7, i) for i in range(64)}) | |
| async def test_incomplete_original_group_not_filled_with_fusions(self): | |
| import integration | |
| builder = integration.ComponentGroupBuilder(lambda: None, 64, initial_state='parent', group_slot=0, logical_step=1) | |
| with self.assertRaises(RuntimeError): | |
| await builder.compute_group_rewards([], [SimpleNamespace(sampler=None)]) | |
| def test_training_config_unchanged(self): | |
| from train_api import build_config | |
| cfg = build_config(Path('/tmp/unused-offline-test')) | |
| self.assertEqual((cfg.num_epochs, cfg.lora_rank, cfg.num_substeps), (9, 32, 1)) | |
| self.assertEqual((cfg.adv_estimator, cfg.loss_fn), ('entropic_adaptive_beta', 'importance_sampling')) | |
| self.assertEqual((cfg.kl_penalty_coef, cfg.learning_rate, cfg.temperature), (.1, 4e-5, 1.)) | |
| self.assertTrue(cfg.remove_constant_reward_groups) | |
| self.assertEqual(cfg.dataset_builder.config.group_size * cfg.dataset_builder.config.batch_size, 512) | |
| self.assertFalse(any([cfg.trace_credit_weight, cfg.factorized_action_credit, cfg.breakthrough_replay_enabled, | |
| cfg.descendant_credit_enabled, cfg.bellman_relabel_enabled, cfg.hindsight_shortcut_enabled])) | |
| def test_native_advantages_remain_finite_with_sparse_successes(self): | |
| from ttt_discover.rl.advantages import compute_group_advantages | |
| rewards = [0.] * 48 + [.2] * 14 + [.25, .3] | |
| advantages = compute_group_advantages(rewards, 'entropic_adaptive_beta') | |
| self.assertTrue(torch.isfinite(advantages).all()) | |
| self.assertGreater(advantages[-1], advantages[0]) | |
| if __name__ == '__main__': | |
| import socket | |
| import sys | |
| suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__]) | |
| with patch.object(socket.socket, 'connect', side_effect=RuntimeError('Network disabled during offline tests')): | |
| result = unittest.TextTestRunner(verbosity=2).run(suite) | |
| report = dict(tests=result.testsRun, failures=len(result.failures), errors=len(result.errors), | |
| success=result.wasSuccessful(), cuda_initialized=torch.cuda.is_initialized(), | |
| network_disabled=True, actual_paid_api_calls=0) | |
| if os.environ.get('KERNEL_TEST_RESULT_PATH'): | |
| atomic(os.environ['KERNEL_TEST_RESULT_PATH'], report) | |
| print('OFFLINE_TEST_RESULT', json.dumps(report), flush=True) | |
| raise SystemExit(0 if result.wasSuccessful() else 1) | |
Xet Storage Details
- Size:
- 22.1 kB
- Xet hash:
- 64486509649fad6516eaebe9f8e8a292894b186e0ecc20ba6d771cc893ca833a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.