zefancainlp's picture
download
raw
22.1 kB
"""No-network tests. All clients in these tests are local fakes."""
from __future__ import annotations
import ast
import asyncio
import copy
from dataclasses import dataclass
import json
import os
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import tinker
import torch
from api_transport import Ledger, ROLLOUT_SLOT, SamplingProxy, RequestSafetyError, atomic, hash_input, sample_payload, cached_sample
from component_core import (build_candidates, gate_layout, metadata_specialize, observed_projection_plans,
profile_contract, sha, structural_key, transfer_projection_plan, verify_uniform_mask_default,
component_fingerprint, normalized_consumer, helpers)
def observed(prefix=5., suffix=5., layout='BNNH4'):
cases = []
for index, n in enumerate([256, 768, 256, 512, 1024, 768, 1024]):
shape = dict(BHNN3=[128, n, n], BHNN4=[1, 128, n, n], BNNH4=[1, n, n, 128], MH2=[n * n, 128])[layout]
cases.append(dict(B=1, N=n, H=128, mid=dict(shape=[128, n, n], dtype='torch.float16'),
gate=dict(shape=shape, dtype='torch.float16'), prefix_us=[prefix] * 3, suffix_us=[suffix] * 3))
return dict(complete=True, cases=cases)
SOURCE = '''import torch
def change_front(x):
return x + 0
def change_tail(x):
return x - 0
def batched_matmul_triton(a,b):
return torch.bmm(a,b.transpose(1,2))
def custom_kernel(data):
inp, mask, weights, cfg = data
dim = cfg['dim']
hidden_dim = cfg['hidden_dim']
B, N, _, _ = inp.shape
device = inp.device
out_gate = inp[..., :hidden_dim]
x = change_front(inp).reshape(B*hidden_dim,N,N)
hidden = batched_matmul_triton(x,x)
return change_tail(hidden).reshape(B,N,N,hidden_dim) * out_gate
'''
class CoreTests(unittest.TestCase):
def test_all_four_layouts(self):
for layout in ['BHNN3', 'BHNN4', 'BNNH4', 'MH2']:
self.assertEqual(profile_contract(observed(layout=layout)), layout)
def test_ambiguous_layout_not_guessed(self):
with self.assertRaises(ValueError):
gate_layout([1, 128, 128, 128], 1, 128, 128)
def test_incomplete_profile_rejected(self):
p = observed()
p['cases'].pop()
with self.assertRaises(ValueError):
profile_contract(p)
def test_dtype_rejected(self):
p = observed()
p['cases'][0]['mid']['dtype'] = 'torch.float32'
with self.assertRaises(ValueError):
profile_contract(p)
def test_changed_layout_policy_rejected(self):
p = observed()
p['cases'][0]['gate']['shape'] = [1, 128, 256, 256]
with self.assertRaises(ValueError):
profile_contract(p)
def test_negative_component_time_rejected(self):
p = observed()
p['cases'][0]['prefix_us'][0] = -1
with self.assertRaises(ValueError):
profile_contract(p)
def test_mid_layout_rejected(self):
p = observed()
p['cases'][0]['mid']['shape'] = [1, 256, 256, 128]
with self.assertRaises(ValueError):
profile_contract(p)
def test_mask_correction_is_only_default(self):
original = "def custom_kernel(data):\n x = data[3].get('nomask', True)\n return x\n"
fixed, n = verify_uniform_mask_default(original)
self.assertEqual(n, 1)
self.assertIn("get('nomask', False)", fixed)
again, n = verify_uniform_mask_default(fixed)
self.assertEqual(n, 0)
self.assertEqual(again, fixed)
def test_joint_search_keeps_individually_worse_candidates(self):
parent = dict(id='p', code=SOURCE, score_us=100., profile=observed())
a = dict(id='a', code=SOURCE.replace('x + 0', 'x + 1'), score_us=150., profile=observed(prefix=1.))
b = dict(id='b', code=SOURCE.replace('x - 0', 'x - 1'), score_us=160., profile=observed(suffix=1.))
programs, audit = build_candidates(parent, [a, b], per_side=2)
self.assertTrue(any(p['front_id'] == 'a' and p['tail_id'] == 'b' for p in programs))
self.assertLessEqual(len(programs), 9)
self.assertEqual(a['score_us'], 150.)
for p in programs:
compile(p['code'], '<joint>', 'exec')
def test_no_borrowed_components_when_only_one_source(self):
parent = dict(id='p', code=SOURCE, score_us=100., profile=observed())
programs, audit = build_candidates(parent, [], per_side=6)
self.assertEqual(programs, [])
self.assertEqual(audit['input_programs'], 1)
def test_no_original_mutation(self):
parent = dict(id='p', code=SOURCE, score_us=100., profile=observed())
children = [dict(id='a', code=SOURCE.replace('x + 0', 'x + 1'), score_us=90., profile=observed())]
before = copy.deepcopy((parent, children))
build_candidates(parent, children)
self.assertEqual(before, (parent, children))
def test_no_numeric_changes_from_scalar_specialization(self):
source = '''import triton
import triton.language as tl
@triton.jit
def k(C, N):
tl.store(C + tl.arange(0,16),N)
def custom_kernel(data):
x = data[0]
B,N,D,H = x.shape
C = x
k[(1,)](C,N)
return C
'''
fixed, edits = metadata_specialize(source)
tree = ast.parse(fixed)
k = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'k')
self.assertIsNone(k.args.args[0].annotation)
self.assertEqual(ast.unparse(k.args.args[1].annotation), 'tl.constexpr')
before = next(n for n in ast.parse(source).body if isinstance(n, ast.FunctionDef) and n.name == 'k')
self.assertEqual(ast.dump(ast.Module(body=before.body, type_ignores=[])), ast.dump(ast.Module(body=k.body, type_ignores=[])))
def test_rebound_scalar_cannot_be_marked_constant(self):
source = '''import triton
import triton.language as tl
@triton.jit
def k(C):
tl.store(C,0)
def custom_kernel(data):
C = 3
C = data[0]
k[(1,)](C)
return C
'''
fixed, edits = metadata_specialize(source)
self.assertEqual(edits, [])
def test_docstring_only_dedup(self):
self.assertEqual(structural_key(SOURCE), structural_key('"a comment"\n' + SOURCE))
def test_generated_namespaces_are_not_new_components(self):
tools, _, _ = helpers()
tree, names = tools.namespaced(ast.parse(SOURCE), 'component_front_')
next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == names['custom_kernel']).name = 'custom_kernel'
# The cut recognizer deliberately supports only its two proven BMM
# entry names. Test namespace invariance within that admitted ABI,
# rather than accidentally constructing an unsupported contraction.
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == names['batched_matmul_triton']:
node.name = 'batched_matmul_triton'
if isinstance(node, ast.Name) and node.id == names['batched_matmul_triton']:
node.id = 'batched_matmul_triton'
namespaced = ast.unparse(ast.fix_missing_locations(tree))
for side in ['front', 'tail']:
self.assertEqual(component_fingerprint(SOURCE, tools, side), component_fingerprint(namespaced, tools, side))
def test_referenced_global_constants_are_not_erased_by_dedup(self):
tools, _, _ = helpers()
source = 'MY_GLOBAL = 1\n' + SOURCE.replace('x + 0', 'x + MY_GLOBAL')
other = source.replace('MY_GLOBAL = 1', 'MY_GLOBAL = 2')
self.assertNotEqual(component_fingerprint(source, tools, 'front'), component_fingerprint(other, tools, 'front'))
def test_own_old_gate_adapter_is_peeled_before_recombination(self):
tools, _, _ = helpers()
source = SOURCE.replace('return change_tail(hidden)',
'_component_consumer_out_gate = out_gate.permute(0,3,1,2).contiguous()\n return change_tail(hidden)')
source = source.replace('* out_gate\n', '* _component_consumer_out_gate\n')
profile = observed()
for case in profile['cases']:
case['gate_aliases'] = {'_component_consumer_out_gate': {'shape': [1, 128, case['N'], case['N']], 'dtype': 'torch.float16'}}
template, layout, audit = normalized_consumer(source, profile, tools)
self.assertEqual(layout, 'BHNN4')
self.assertEqual(audit['peeled_gate_adapters'], 1)
self.assertNotIn('_component_consumer_out_gate', template)
self.assertIn('* out_gate', template)
class FakeSampling:
def __init__(self):
self.calls = 0
async def sample_async(self, **kwargs):
self.calls += 1
return cached_sample(dict(sequences=[dict(tokens=[11, 12], logprobs=[-.1, -.2], stop_reason='stop')]))
async def compute_logprobs_async(self, model_input, **kwargs):
self.calls += 1
return [None, -.2]
class CacheTests(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.ledger = Ledger(self.tmp.name, enabled=True)
self.ledger.logical_step = 1
self.fake = FakeSampling()
self.proxy = SamplingProxy(self.fake, self.ledger, policy='boundary-0')
self.prompt = tinker.ModelInput.from_ints([1, 2])
self.params = tinker.SamplingParams(max_tokens=8, temperature=1.)
self.slot = ROLLOUT_SLOT.set((1, 0, 0))
self.env = patch.dict(os.environ, {'TTT_ALLOW_PAID_KERNEL_API10': '1'})
self.env.start()
def tearDown(self):
ROLLOUT_SLOT.reset(self.slot)
self.env.stop()
self.tmp.cleanup()
async def test_exact_response_reused_without_second_model_call(self):
first = await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
resumed = SamplingProxy(self.fake, self.ledger, policy='boundary-0')
second = await resumed.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
self.assertEqual(sample_payload(first), sample_payload(second))
self.assertEqual(self.fake.calls, 1)
async def test_third_phase_refused(self):
for _ in range(2):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
with self.assertRaises(RequestSafetyError):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
self.assertEqual(self.fake.calls, 2)
async def test_disabled_paid_gate(self):
self.ledger.enabled = False
with self.assertRaises(RequestSafetyError):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
self.assertEqual(self.fake.calls, 0)
async def test_no_untracked_generation(self):
ROLLOUT_SLOT.set(None)
with self.assertRaises(RequestSafetyError):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
async def test_no_more_than_64_per_group(self):
ROLLOUT_SLOT.set((1, 0, 64))
with self.assertRaises(RequestSafetyError):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
async def test_no_more_than_8_groups(self):
ROLLOUT_SLOT.set((1, 8, 0))
with self.assertRaises(RequestSafetyError):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
async def test_unknown_paid_request_not_retried(self):
identity = dict(slot=[1, 0, 0], phase=1, policy='boundary-0', prompt_sha256=hash_input(self.prompt),
settings=self.params.model_dump(mode='json'), kwargs={})
_, pending = self.ledger.cache_paths(json.dumps(identity, sort_keys=True))
atomic(pending, dict(identity=identity))
with self.assertRaises(RequestSafetyError):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
self.assertEqual(self.fake.calls, 0)
async def test_reference_not_a_generator(self):
proxy = SamplingProxy(self.fake, self.ledger, policy='base', reference=True)
with self.assertRaises(RequestSafetyError):
await proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
async def test_reference_cache_reused(self):
proxy = SamplingProxy(self.fake, self.ledger, policy='base', reference=True)
a = await proxy.compute_logprobs_async(self.prompt)
b = await proxy.compute_logprobs_async(self.prompt)
self.assertEqual(a, b)
self.assertEqual(self.fake.calls, 1)
async def test_policy_boundary_is_part_of_identity(self):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
changed = SamplingProxy(self.fake, self.ledger, policy='boundary-1')
with self.assertRaises(RequestSafetyError):
await changed.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
self.assertEqual(self.fake.calls, 1)
async def test_same_slot_different_prompt_cannot_be_resampled(self):
await self.proxy.sample_async(prompt=self.prompt, num_samples=1, sampling_params=self.params)
changed = SamplingProxy(self.fake, self.ledger, policy='boundary-0')
with self.assertRaises(RequestSafetyError):
await changed.sample_async(prompt=tinker.ModelInput.from_ints([3, 4]), num_samples=1, sampling_params=self.params)
self.assertEqual(self.fake.calls, 1)
class IntegrationTests(unittest.IsolatedAsyncioTestCase):
async def test_native_512_sample_update_uses_no_synthetic_program_tokens(self):
from api_transport import TrainingProxy
from ttt_discover.rl.types import Trajectory, TrajectoryGroup, Transition
from ttt_discover.tinker_utils.completers import TokensWithLogprobs
from ttt_discover.rl.data_processing import assemble_training_data
from ttt_discover.rl.advantages import compute_group_advantages
from ttt_discover.rl.train import train_step, incorporate_kl_penalty
groups, advantages = [], []
for group in range(8):
trajectories = []
rewards = []
for i in range(64):
action = [10 + group, 100 + i]
transition = Transition(ob=tinker.ModelInput.from_ints([1, 2, 3]),
ac=TokensWithLogprobs(tokens=action, maybe_logprobs=[-.2, -.3], maybe_mask=[1., 1.]),
reward=i / 64., episode_done=True, metrics={})
trajectories.append(Trajectory(transitions=[transition], final_ob=tinker.ModelInput.empty()))
rewards.append(i / 64.)
groups.append(TrajectoryGroup(trajectories_G=trajectories, final_rewards_G=[0.] * 64, metrics_G=[{} for _ in range(64)]))
advantages.append(compute_group_advantages(rewards, 'entropic_adaptive_beta'))
data, metadata = assemble_training_data(groups, advantages)
self.assertEqual(len(data), 512)
self.assertEqual(len(metadata), 512)
self.assertEqual(sum(float(d.loss_fn_inputs['mask'].to_torch().sum()) for d in data), 1024.)
captured = []
class Future:
def __init__(self, result): self.value = result
async def result_async(self): return self.value
class FakeTraining:
async def forward_backward_async(self, rows, **kwargs):
captured.append(('fwb', rows, kwargs))
self.rows = rows
return Future(SimpleNamespace(loss_fn_outputs=[{'logprobs': tinker.TensorData.from_torch(torch.zeros(d.model_input.length))} for d in rows]))
async def optim_step_async(self, params):
captured.append(('adam', params))
return Future(SimpleNamespace())
class Reference:
async def compute_logprobs_async(self, model_input):
return [None] + [-.1] * (model_input.length - 1)
with tempfile.TemporaryDirectory() as tmp, patch.dict(os.environ, {'TTT_ALLOW_PAID_KERNEL_API10': '1'}):
ledger = Ledger(tmp, enabled=True)
ledger.logical_step = 1
training = TrainingProxy(FakeTraining(), SimpleNamespace(ledger=ledger))
original_targets = [d.loss_fn_inputs['target_tokens'].to_torch().clone() for d in data]
await incorporate_kl_penalty(data, Reference(), .1)
trained = await train_step(data, training, learning_rate=4e-5, num_substeps=1, loss_fn='importance_sampling')
self.assertEqual(len(trained), 512)
self.assertEqual([r[0] for r in captured], ['fwb', 'adam'])
self.assertEqual(len(captured[0][1]), 512)
self.assertTrue(all('mask' not in d.loss_fn_inputs for d in captured[0][1]))
self.assertTrue(all(torch.equal(before, after.loss_fn_inputs['target_tokens'].to_torch()) for before, after in zip(original_targets, data)))
async def test_64_original_trajectories_survive_search_unchanged(self):
import integration
called = []
runtime = SimpleNamespace(fuse_group=lambda *args: called.append(args))
trajectories = [SimpleNamespace(transitions=[SimpleNamespace(ac=SimpleNamespace(tokens=[i, i + 1]), reward=i / 64.)]) for i in range(64)]
envs = [SimpleNamespace(sampler='sampler') for _ in range(64)]
builder = integration.ComponentGroupBuilder(lambda: None, 64, initial_state='parent', group_slot=0, logical_step=1)
with patch.object(integration, 'RUNTIME', runtime):
rewards = await builder.compute_group_rewards(trajectories, envs)
self.assertEqual(rewards, [(0., {})] * 64)
self.assertEqual(len(called), 1)
self.assertEqual([t.transitions[0].reward for t in trajectories], [i / 64. for i in range(64)])
async def test_fusion_admission_is_on_the_native_event_loop(self):
import integration
import threading
loop_thread = threading.get_ident()
node = SimpleNamespace(code='synthetic search node, not a trajectory')
updates = []
sampler = SimpleNamespace(update_states=lambda *args, **kwargs: updates.append((threading.get_ident(), args)))
def search(*args):
self.assertNotEqual(threading.get_ident(), loop_thread)
return node
runtime = SimpleNamespace(fuse_group=search)
trajectories = [SimpleNamespace(transitions=[SimpleNamespace(ac=SimpleNamespace(tokens=[i]), reward=float(i))]) for i in range(64)]
envs = [SimpleNamespace(sampler=sampler) for _ in range(64)]
builder = integration.ComponentGroupBuilder(lambda: None, 64, initial_state='parent', group_slot=0, logical_step=1)
with patch.object(integration, 'RUNTIME', runtime):
rewards = await builder.compute_group_rewards(trajectories, envs)
self.assertEqual(len(updates), 1)
self.assertEqual(updates[0][0], loop_thread)
self.assertIs(updates[0][1][0][0], node)
self.assertEqual(rewards, [(0., {})] * 64)
async def test_every_episode_has_a_distinct_stable_slot(self):
import integration
builder = integration.ComponentGroupBuilder(lambda: SimpleNamespace(), 64, initial_state='parent', group_slot=7, logical_step=9)
envs = await builder.make_envs()
self.assertEqual({e._component_slot for e in envs}, {(9, 7, i) for i in range(64)})
async def test_incomplete_original_group_not_filled_with_fusions(self):
import integration
builder = integration.ComponentGroupBuilder(lambda: None, 64, initial_state='parent', group_slot=0, logical_step=1)
with self.assertRaises(RuntimeError):
await builder.compute_group_rewards([], [SimpleNamespace(sampler=None)])
def test_training_config_unchanged(self):
from train_api import build_config
cfg = build_config(Path('/tmp/unused-offline-test'))
self.assertEqual((cfg.num_epochs, cfg.lora_rank, cfg.num_substeps), (9, 32, 1))
self.assertEqual((cfg.adv_estimator, cfg.loss_fn), ('entropic_adaptive_beta', 'importance_sampling'))
self.assertEqual((cfg.kl_penalty_coef, cfg.learning_rate, cfg.temperature), (.1, 4e-5, 1.))
self.assertTrue(cfg.remove_constant_reward_groups)
self.assertEqual(cfg.dataset_builder.config.group_size * cfg.dataset_builder.config.batch_size, 512)
self.assertFalse(any([cfg.trace_credit_weight, cfg.factorized_action_credit, cfg.breakthrough_replay_enabled,
cfg.descendant_credit_enabled, cfg.bellman_relabel_enabled, cfg.hindsight_shortcut_enabled]))
def test_native_advantages_remain_finite_with_sparse_successes(self):
from ttt_discover.rl.advantages import compute_group_advantages
rewards = [0.] * 48 + [.2] * 14 + [.25, .3]
advantages = compute_group_advantages(rewards, 'entropic_adaptive_beta')
self.assertTrue(torch.isfinite(advantages).all())
self.assertGreater(advantages[-1], advantages[0])
if __name__ == '__main__':
import socket
import sys
suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__])
with patch.object(socket.socket, 'connect', side_effect=RuntimeError('Network disabled during offline tests')):
result = unittest.TextTestRunner(verbosity=2).run(suite)
report = dict(tests=result.testsRun, failures=len(result.failures), errors=len(result.errors),
success=result.wasSuccessful(), cuda_initialized=torch.cuda.is_initialized(),
network_disabled=True, actual_paid_api_calls=0)
if os.environ.get('KERNEL_TEST_RESULT_PATH'):
atomic(os.environ['KERNEL_TEST_RESULT_PATH'], report)
print('OFFLINE_TEST_RESULT', json.dumps(report), flush=True)
raise SystemExit(0 if result.wasSuccessful() else 1)

Xet Storage Details

Size:
22.1 kB
·
Xet hash:
64486509649fad6516eaebe9f8e8a292894b186e0ecc20ba6d771cc893ca833a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.