Buckets:
| """Add search-only component nodes; keep the ordinary single-anchor RL batch. | |
| No synthetic trajectory, fusion reward backfill, policy-gradient rewrite, or | |
| external paper implementation enters training. Ordinary PUCT admits a validated | |
| fusion exactly as one additional evaluated child, including its normal visit | |
| accounting; the PUCT equation and original episode accounting are unchanged. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| from dataclasses import dataclass, replace | |
| import hashlib | |
| import json | |
| import math | |
| from pathlib import Path | |
| import threading | |
| import chz | |
| from api_transport import ROLLOUT_SLOT, atomic | |
| from component_core import build_candidates, metadata_specialize, observed_projection_plans, sha, transfer_projection_plan | |
| from gpu_pool import JudgePool | |
| from examples.gpu_mode.env import (SingleAnchorTriMulEnv, SingleAnchorTriMulState, | |
| SingleAnchorTriMulRewardEvaluator, invalid) | |
| from examples.gpu_mode.single_anchor import merge_single_anchor | |
| from ttt_discover.rl.types import ProblemGroupBuilder | |
| from ttt_discover.rl.types import RLDatasetBuilder | |
| from ttt_discover.tinker_utils.dataset_builder import SingleProblemDataset, SingleProblemDatasetBuilder | |
| from ttt_discover.tinker_utils.dataset_builder import DatasetConfig | |
| from ttt_discover.tinker_utils.program_graph import ProgramGraphError | |
| RUNTIME = None | |
| class FusionTriMulState(SingleAnchorTriMulState): | |
| def __init__(self, *args, component_profile=None, fusion_history=None, **kwargs): | |
| super().__init__(*args, **kwargs) | |
| self.component_profile = component_profile or {} | |
| self.fusion_history = list(fusion_history or []) | |
| def to_dict(self): | |
| return {**super().to_dict(), 'component_profile': self.component_profile, 'fusion_history': self.fusion_history} | |
| class ComponentRewardEvaluator(SingleAnchorTriMulRewardEvaluator): | |
| def get_reward(self, code, state): | |
| try: | |
| merged = merge_single_anchor(state.code, code) | |
| except (ProgramGraphError, SyntaxError, ValueError) as error: | |
| return invalid('Invalid single-anchor patch: ' + str(error)) | |
| if '@triton.jit' not in merged.source or 'identity' in merged.source: | |
| return invalid('Original upstream Triton/identity guard failed') | |
| slot = getattr(state, '_component_slot', None) | |
| if slot is None: | |
| raise RuntimeError('Missing stable original-rollout identity') | |
| result = RUNTIME.pool.evaluate(merged.source, key='original/' + '/'.join(map(str, slot))) | |
| metrics = {'component_profile': result.get('component_profile', {}), | |
| 'trimul/request_id': result['request_id'], 'trimul/gpu_index': result['gpu_index'], | |
| 'trimul/gpu_uuid': result['gpu_uuid'], 'trimul/true_boolean_checks': len(result.get('boolean_checks', []))} | |
| return dict(reward=result['reward'], correctness=result['correctness'], raw_score=result['raw_score'], | |
| msg=f"Official TriMul runtime: {result.get('score_us')} us ({result['status']})", result_construction=[], stdout='', metrics=metrics) | |
| class ComponentTriMulEnv(SingleAnchorTriMulEnv): | |
| state_type = FusionTriMulState | |
| reward_function = ComponentRewardEvaluator | |
| # Prompt/get_question, patch grammar, reward transform, and single anchor are inherited. | |
| backup_descendant_values = False | |
| max_policy_survivors = False | |
| dedupe_by_semantics = False | |
| dedupe_semantics_within_parent = False | |
| def create_initial_state(cls, problem_type): | |
| if problem_type != 'trimul' or RUNTIME is None: | |
| raise ValueError('Prepared, independently verified TriMul Step 0 required') | |
| row = dict(RUNTIME.root_row) | |
| root = cls.state_type.from_root(row) | |
| root.component_profile = RUNTIME.root_profile | |
| return root | |
| async def initial_observation(self): | |
| if not hasattr(self, '_component_slot'): | |
| raise RuntimeError('Environment group did not assign an exact rollout slot') | |
| ROLLOUT_SLOT.set(self._component_slot) | |
| self.state = self.state_type.from_dict(self.initial_state.to_dict()) | |
| self.state._component_slot = self._component_slot | |
| return await super().initial_observation() | |
| def _create_next_state(self, step_idx, parsed_code, outs): | |
| state = super()._create_next_state(step_idx + 1, parsed_code, outs) | |
| state.component_profile = (outs.metrics or {}).get('component_profile', {}) | |
| state.fusion_history = list(self.initial_state.fusion_history) | |
| self._component_child = state | |
| return state | |
| class ComponentGroupBuilder(ProblemGroupBuilder): | |
| initial_state: object = None | |
| group_slot: int = -1 | |
| logical_step: int = -1 | |
| async def make_envs(self): | |
| environments = await super().make_envs() | |
| if len(environments) != 64 or not 0 <= self.group_slot < 8 or not 1 <= self.logical_step <= 9: | |
| raise RuntimeError('Exact 8x64 budget mismatch') | |
| for index, env in enumerate(environments): | |
| env._component_slot = (self.logical_step, self.group_slot, index) | |
| return environments | |
| async def compute_group_rewards(self, trajectory_group, env_group): | |
| # Verify that search cannot change a trajectory's sampled action or reward. | |
| before = [(tuple(tuple(t.ac.tokens) for t in tr.transitions), tuple(t.reward for t in tr.transitions)) for tr in trajectory_group] | |
| children = [e._component_child for e in env_group if hasattr(e, '_component_child')] | |
| if len(trajectory_group) != 64: | |
| raise RuntimeError('Incomplete original group; do not replace missing samples with fusions') | |
| sampler = env_group[0].sampler | |
| accepted = await asyncio.to_thread(RUNTIME.fuse_group, self.logical_step, self.group_slot, self.initial_state, children) | |
| # Native episode admissions also run on this event loop. Do not mutate | |
| # PUCT's counters/archive from the background GPU-search thread. | |
| if accepted is not None: | |
| sampler.update_states([accepted], [self.initial_state], save=False) | |
| after = [(tuple(tuple(t.ac.tokens) for t in tr.transitions), tuple(t.reward for t in tr.transitions)) for tr in trajectory_group] | |
| if before != after: | |
| raise RuntimeError('Fusion search modified the RL training batch') | |
| return [(0.0, {}) for _ in trajectory_group] | |
| ComponentTriMulEnv.group_builder_type = ComponentGroupBuilder | |
| class ComponentDataset(SingleProblemDataset): | |
| def get_batch(self, index): | |
| logical = int(self.sampler._current_step) + 1 | |
| RUNTIME.ledger.logical_step = logical | |
| path = RUNTIME.root / 'run/parent_batches' / (f'{logical:02d}.json') | |
| if path.exists(): | |
| saved = json.loads(path.read_text()) | |
| if saved['logical_step'] != logical or len(saved['parents']) != 8: | |
| raise RuntimeError('Uncommitted batch parent manifest is inconsistent') | |
| states = [self.config.env_type.state_type.from_dict(row) for row in saved['parents']] | |
| builders = [self._make_env_group_builder(state, 64) for state in states] | |
| else: | |
| builders = super().get_batch(index) | |
| atomic(path, dict(logical_step=logical, parents=[b.initial_state.to_dict() for b in builders])) | |
| result = [replace(b, group_slot=i, logical_step=logical) for i, b in enumerate(builders)] | |
| # Check every selected context before ANY group can spend API tokens. | |
| from ttt_discover.tinker_utils.misc_utils import get_tokenizer | |
| tokenizer = get_tokenizer(self.config.model_name_for_tokenizer) | |
| for builder in result: | |
| env = builder.env_thunk() | |
| prompt = env.renderer.build_generation_prompt([{'role': 'user', 'content': env.get_question()}]) | |
| if prompt.length >= 26000: | |
| raise RuntimeError('A PUCT-selected parent exceeds the unchanged phase-1 context budget; no paid batch was launched') | |
| return result | |
| def flush(self, step=None): | |
| super().flush(step=step) | |
| if step is None: | |
| return | |
| records = [] | |
| for group in range(8): | |
| path = RUNTIME.root / 'run/component_search' / f's{step:02d}_g{group:02d}' / 'complete.json' | |
| if not path.exists(): | |
| raise RuntimeError('Cannot commit a sampler without all eight component-search group receipts') | |
| records.append(json.loads(path.read_text())) | |
| best = max((s for s in self.sampler._states if s.value is not None), key=lambda s: s.value) | |
| atomic(RUNTIME.root / 'run/step_summaries' / f'{step:02d}.json', | |
| dict(logical_step=step, status='sampled_not_trained', original_rollouts=512, | |
| accepted_fusions=sum(r['accepted_state'] is not None for r in records), | |
| screened_fusion_programs=sum(r['candidate_count'] for r in records), | |
| full_fusion_evaluations=sum(r['full_fusion_evaluations'] for r in records), | |
| best_us=-best.value, best_state_id=best.id, best_program_sha256=sha(best.code), | |
| best_has_fusion=bool(getattr(best, 'fusion_history', [])), | |
| training_data='original_rollouts_only', reward_backfills=0)) | |
| class ComponentDatasetBuilder(RLDatasetBuilder): | |
| config: DatasetConfig | |
| async def __call__(self): | |
| ordinary = await SingleProblemDatasetBuilder(config=self.config)() | |
| return ComponentDataset(ordinary.config, ordinary.renderer, ordinary.sampler) | |
| class ComponentEngine: | |
| def __init__(self, root, ledger, *, gpus=tuple(range(8)), per_side=6, finalists=2): | |
| self.root, self.ledger = Path(root), ledger | |
| self.pool = JudgePool(root, gpus=gpus) | |
| prepared = json.loads((self.root / 'control/validated_root.json').read_text()) | |
| self.root_row, self.root_profile = prepared['root'], prepared['profile'] | |
| self.per_side, self.finalists = per_side, finalists | |
| self.lock = threading.Lock() | |
| def record(state): | |
| return dict(id=state.id, code=state.code, score_us=-float(state.value), profile=state.component_profile) | |
| def fuse_group(self, logical_step, group_slot, parent, children): | |
| directory = self.root / 'run/component_search' / (f's{logical_step:02d}_g{group_slot:02d}') | |
| directory.mkdir(parents=True, exist_ok=True) | |
| completed = directory / 'complete.json' | |
| if completed.exists(): | |
| data = json.loads(completed.read_text()) | |
| if data.get('accepted_state'): | |
| return FusionTriMulState.from_dict(data['accepted_state']) | |
| return None | |
| records = [self.record(s) for s in children] | |
| root = self.record(parent) | |
| candidates, audit = build_candidates(root, records, per_side=self.per_side) | |
| # One whole already-observed plan, not a Cartesian grid of fresh knobs. | |
| plans = observed_projection_plans([root, *records], limit=1) | |
| variants, seen = [], set() | |
| for record in candidates: | |
| forms = [(record['code'], 'original', [])] | |
| # The adapter recognizes an existing fused-consumer ABI. It imports | |
| # no paper kernel: the numerical helper still comes from this group. | |
| if record['audit']['front_gate_layout'] != record['audit']['tail_gate_layout']: | |
| try: | |
| from prepare_strided_fusion import adapt | |
| lowered, stride_audit = adapt(record['code'], record['audit']['front_gate_layout'], 'paper') | |
| forms.append((lowered, 'affine_gate_stride_lowering', [stride_audit])) | |
| except (StopIteration, ValueError, AssertionError): | |
| pass | |
| for source, mode, edits in list(forms): | |
| specialized, changes = metadata_specialize(source) | |
| if changes: | |
| forms.append((specialized, mode + '_proven_scalar_metadata', [*edits, *changes])) | |
| for source, compiler_mode, edits in list(forms): | |
| for plan in plans: | |
| transferred, applied = transfer_projection_plan(source, plan) | |
| if applied: | |
| forms.append((transferred, compiler_mode + '_existing_plan_' + '-'.join(map(str, plan)), edits)) | |
| for source, mode, edits in forms: | |
| fingerprint = sha(source) | |
| if fingerprint in seen: | |
| continue | |
| seen.add(fingerprint) | |
| variants.append({**record, 'label': 'joint_' + fingerprint[:16], 'code': source, 'sha256': fingerprint, | |
| 'compiler_mode': mode, 'metadata_edits': edits}) | |
| assert len(variants) <= (self.per_side + 1) ** 2 * 8 | |
| packet = dict(logical_step=logical_step, group_slot=group_slot, parent=root, | |
| original_valid_children=records, assembly_audit=audit, observed_plans=plans, candidates=variants) | |
| atomic(directory / 'manifest.json', packet) | |
| accepted, formal = None, [] | |
| if variants: | |
| best_original = min([root, *records], key=lambda r: r['score_us']) | |
| controls = [dict(label='parent_control', code=root['code'], kind='control')] | |
| if best_original['id'] != root['id']: | |
| controls.append(dict(label='best_original_control', code=best_original['code'], kind='control')) | |
| screen = self.pool.screen([*controls, *variants], key=f'search/{logical_step}/{group_slot}') | |
| atomic(directory / 'screen.json', screen) | |
| lookup = {r['label']: r for r in variants} | |
| for control in controls: | |
| result = next(r for r in screen['results'] if r['label'] == control['label']) | |
| if not result['valid']: | |
| raise RuntimeError('A previously valid control failed component screening') | |
| ranked = sorted([r for r in screen['results'] if r['valid'] and r['label'] in lookup], key=lambda r: r['geometric_us']) | |
| for choice in ranked[:self.finalists]: | |
| candidate = lookup[choice['label']] | |
| result = self.pool.evaluate(candidate['code'], key=f'fusion/{logical_step}/{group_slot}/{choice["label"]}') | |
| formal.append(dict(candidate=candidate, result=result)) | |
| improving = [r for r in formal if r['result']['correctness'] > 0 and r['result']['score_us'] < best_original['score_us']] | |
| if improving: | |
| winner = min(improving, key=lambda r: r['result']['score_us']) | |
| provenance = dict(logical_step=logical_step, group_slot=group_slot, | |
| candidate_sha256=winner['candidate']['sha256'], | |
| front_id=winner['candidate']['front_id'], tail_id=winner['candidate']['tail_id'], | |
| result_request_id=winner['result']['request_id'], | |
| source='script_assembly_not_policy_sample', score_us=winner['result']['score_us']) | |
| accepted = FusionTriMulState(timestep=logical_step, construction=[], code=winner['candidate']['code'], | |
| value=-winner['result']['score_us'], terminal_value=-winner['result']['score_us'], | |
| step0_source=parent.step0_source, bootstrap_winner_id=parent.bootstrap_winner_id, | |
| patch_history=list(parent.patch_history), fusion_history=[*parent.fusion_history, provenance], | |
| component_profile=winner['result'].get('component_profile', {})) | |
| atomic(completed, dict(logical_step=logical_step, group_slot=group_slot, candidate_count=len(variants), | |
| original_valid_children=len(records), full_fusion_evaluations=len(formal), | |
| raw_original_best_us=min([root['score_us'], *(r['score_us'] for r in records)]), | |
| best_fusion_us=min([r['result']['score_us'] for r in formal if r['result']['correctness']] or [None]), | |
| accepted_state=accepted.to_dict() if accepted else None, | |
| training_trajectories_added=0, reward_backfills=0)) | |
| return accepted | |
| def configure(root, ledger, **kwargs): | |
| global RUNTIME | |
| RUNTIME = ComponentEngine(root, ledger, **kwargs) | |
| return RUNTIME | |
Xet Storage Details
- Size:
- 16.4 kB
- Xet hash:
- 7dc7e834ddc4f743a658ee4fbc8ba647aaf3f8870c8a0c2c7e1ee81e90662381
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.