zefancainlp's picture
download
raw
16.4 kB
"""Add search-only component nodes; keep the ordinary single-anchor RL batch.
No synthetic trajectory, fusion reward backfill, policy-gradient rewrite, or
external paper implementation enters training. Ordinary PUCT admits a validated
fusion exactly as one additional evaluated child, including its normal visit
accounting; the PUCT equation and original episode accounting are unchanged.
"""
from __future__ import annotations
import asyncio
from dataclasses import dataclass, replace
import hashlib
import json
import math
from pathlib import Path
import threading
import chz
from api_transport import ROLLOUT_SLOT, atomic
from component_core import build_candidates, metadata_specialize, observed_projection_plans, sha, transfer_projection_plan
from gpu_pool import JudgePool
from examples.gpu_mode.env import (SingleAnchorTriMulEnv, SingleAnchorTriMulState,
SingleAnchorTriMulRewardEvaluator, invalid)
from examples.gpu_mode.single_anchor import merge_single_anchor
from ttt_discover.rl.types import ProblemGroupBuilder
from ttt_discover.rl.types import RLDatasetBuilder
from ttt_discover.tinker_utils.dataset_builder import SingleProblemDataset, SingleProblemDatasetBuilder
from ttt_discover.tinker_utils.dataset_builder import DatasetConfig
from ttt_discover.tinker_utils.program_graph import ProgramGraphError
RUNTIME = None
class FusionTriMulState(SingleAnchorTriMulState):
def __init__(self, *args, component_profile=None, fusion_history=None, **kwargs):
super().__init__(*args, **kwargs)
self.component_profile = component_profile or {}
self.fusion_history = list(fusion_history or [])
def to_dict(self):
return {**super().to_dict(), 'component_profile': self.component_profile, 'fusion_history': self.fusion_history}
class ComponentRewardEvaluator(SingleAnchorTriMulRewardEvaluator):
def get_reward(self, code, state):
try:
merged = merge_single_anchor(state.code, code)
except (ProgramGraphError, SyntaxError, ValueError) as error:
return invalid('Invalid single-anchor patch: ' + str(error))
if '@triton.jit' not in merged.source or 'identity' in merged.source:
return invalid('Original upstream Triton/identity guard failed')
slot = getattr(state, '_component_slot', None)
if slot is None:
raise RuntimeError('Missing stable original-rollout identity')
result = RUNTIME.pool.evaluate(merged.source, key='original/' + '/'.join(map(str, slot)))
metrics = {'component_profile': result.get('component_profile', {}),
'trimul/request_id': result['request_id'], 'trimul/gpu_index': result['gpu_index'],
'trimul/gpu_uuid': result['gpu_uuid'], 'trimul/true_boolean_checks': len(result.get('boolean_checks', []))}
return dict(reward=result['reward'], correctness=result['correctness'], raw_score=result['raw_score'],
msg=f"Official TriMul runtime: {result.get('score_us')} us ({result['status']})", result_construction=[], stdout='', metrics=metrics)
class ComponentTriMulEnv(SingleAnchorTriMulEnv):
state_type = FusionTriMulState
reward_function = ComponentRewardEvaluator
# Prompt/get_question, patch grammar, reward transform, and single anchor are inherited.
backup_descendant_values = False
max_policy_survivors = False
dedupe_by_semantics = False
dedupe_semantics_within_parent = False
@classmethod
def create_initial_state(cls, problem_type):
if problem_type != 'trimul' or RUNTIME is None:
raise ValueError('Prepared, independently verified TriMul Step 0 required')
row = dict(RUNTIME.root_row)
root = cls.state_type.from_root(row)
root.component_profile = RUNTIME.root_profile
return root
async def initial_observation(self):
if not hasattr(self, '_component_slot'):
raise RuntimeError('Environment group did not assign an exact rollout slot')
ROLLOUT_SLOT.set(self._component_slot)
self.state = self.state_type.from_dict(self.initial_state.to_dict())
self.state._component_slot = self._component_slot
return await super().initial_observation()
def _create_next_state(self, step_idx, parsed_code, outs):
state = super()._create_next_state(step_idx + 1, parsed_code, outs)
state.component_profile = (outs.metrics or {}).get('component_profile', {})
state.fusion_history = list(self.initial_state.fusion_history)
self._component_child = state
return state
@dataclass(frozen=True)
class ComponentGroupBuilder(ProblemGroupBuilder):
initial_state: object = None
group_slot: int = -1
logical_step: int = -1
async def make_envs(self):
environments = await super().make_envs()
if len(environments) != 64 or not 0 <= self.group_slot < 8 or not 1 <= self.logical_step <= 9:
raise RuntimeError('Exact 8x64 budget mismatch')
for index, env in enumerate(environments):
env._component_slot = (self.logical_step, self.group_slot, index)
return environments
async def compute_group_rewards(self, trajectory_group, env_group):
# Verify that search cannot change a trajectory's sampled action or reward.
before = [(tuple(tuple(t.ac.tokens) for t in tr.transitions), tuple(t.reward for t in tr.transitions)) for tr in trajectory_group]
children = [e._component_child for e in env_group if hasattr(e, '_component_child')]
if len(trajectory_group) != 64:
raise RuntimeError('Incomplete original group; do not replace missing samples with fusions')
sampler = env_group[0].sampler
accepted = await asyncio.to_thread(RUNTIME.fuse_group, self.logical_step, self.group_slot, self.initial_state, children)
# Native episode admissions also run on this event loop. Do not mutate
# PUCT's counters/archive from the background GPU-search thread.
if accepted is not None:
sampler.update_states([accepted], [self.initial_state], save=False)
after = [(tuple(tuple(t.ac.tokens) for t in tr.transitions), tuple(t.reward for t in tr.transitions)) for tr in trajectory_group]
if before != after:
raise RuntimeError('Fusion search modified the RL training batch')
return [(0.0, {}) for _ in trajectory_group]
ComponentTriMulEnv.group_builder_type = ComponentGroupBuilder
class ComponentDataset(SingleProblemDataset):
def get_batch(self, index):
logical = int(self.sampler._current_step) + 1
RUNTIME.ledger.logical_step = logical
path = RUNTIME.root / 'run/parent_batches' / (f'{logical:02d}.json')
if path.exists():
saved = json.loads(path.read_text())
if saved['logical_step'] != logical or len(saved['parents']) != 8:
raise RuntimeError('Uncommitted batch parent manifest is inconsistent')
states = [self.config.env_type.state_type.from_dict(row) for row in saved['parents']]
builders = [self._make_env_group_builder(state, 64) for state in states]
else:
builders = super().get_batch(index)
atomic(path, dict(logical_step=logical, parents=[b.initial_state.to_dict() for b in builders]))
result = [replace(b, group_slot=i, logical_step=logical) for i, b in enumerate(builders)]
# Check every selected context before ANY group can spend API tokens.
from ttt_discover.tinker_utils.misc_utils import get_tokenizer
tokenizer = get_tokenizer(self.config.model_name_for_tokenizer)
for builder in result:
env = builder.env_thunk()
prompt = env.renderer.build_generation_prompt([{'role': 'user', 'content': env.get_question()}])
if prompt.length >= 26000:
raise RuntimeError('A PUCT-selected parent exceeds the unchanged phase-1 context budget; no paid batch was launched')
return result
def flush(self, step=None):
super().flush(step=step)
if step is None:
return
records = []
for group in range(8):
path = RUNTIME.root / 'run/component_search' / f's{step:02d}_g{group:02d}' / 'complete.json'
if not path.exists():
raise RuntimeError('Cannot commit a sampler without all eight component-search group receipts')
records.append(json.loads(path.read_text()))
best = max((s for s in self.sampler._states if s.value is not None), key=lambda s: s.value)
atomic(RUNTIME.root / 'run/step_summaries' / f'{step:02d}.json',
dict(logical_step=step, status='sampled_not_trained', original_rollouts=512,
accepted_fusions=sum(r['accepted_state'] is not None for r in records),
screened_fusion_programs=sum(r['candidate_count'] for r in records),
full_fusion_evaluations=sum(r['full_fusion_evaluations'] for r in records),
best_us=-best.value, best_state_id=best.id, best_program_sha256=sha(best.code),
best_has_fusion=bool(getattr(best, 'fusion_history', [])),
training_data='original_rollouts_only', reward_backfills=0))
@chz.chz
class ComponentDatasetBuilder(RLDatasetBuilder):
config: DatasetConfig
async def __call__(self):
ordinary = await SingleProblemDatasetBuilder(config=self.config)()
return ComponentDataset(ordinary.config, ordinary.renderer, ordinary.sampler)
class ComponentEngine:
def __init__(self, root, ledger, *, gpus=tuple(range(8)), per_side=6, finalists=2):
self.root, self.ledger = Path(root), ledger
self.pool = JudgePool(root, gpus=gpus)
prepared = json.loads((self.root / 'control/validated_root.json').read_text())
self.root_row, self.root_profile = prepared['root'], prepared['profile']
self.per_side, self.finalists = per_side, finalists
self.lock = threading.Lock()
@staticmethod
def record(state):
return dict(id=state.id, code=state.code, score_us=-float(state.value), profile=state.component_profile)
def fuse_group(self, logical_step, group_slot, parent, children):
directory = self.root / 'run/component_search' / (f's{logical_step:02d}_g{group_slot:02d}')
directory.mkdir(parents=True, exist_ok=True)
completed = directory / 'complete.json'
if completed.exists():
data = json.loads(completed.read_text())
if data.get('accepted_state'):
return FusionTriMulState.from_dict(data['accepted_state'])
return None
records = [self.record(s) for s in children]
root = self.record(parent)
candidates, audit = build_candidates(root, records, per_side=self.per_side)
# One whole already-observed plan, not a Cartesian grid of fresh knobs.
plans = observed_projection_plans([root, *records], limit=1)
variants, seen = [], set()
for record in candidates:
forms = [(record['code'], 'original', [])]
# The adapter recognizes an existing fused-consumer ABI. It imports
# no paper kernel: the numerical helper still comes from this group.
if record['audit']['front_gate_layout'] != record['audit']['tail_gate_layout']:
try:
from prepare_strided_fusion import adapt
lowered, stride_audit = adapt(record['code'], record['audit']['front_gate_layout'], 'paper')
forms.append((lowered, 'affine_gate_stride_lowering', [stride_audit]))
except (StopIteration, ValueError, AssertionError):
pass
for source, mode, edits in list(forms):
specialized, changes = metadata_specialize(source)
if changes:
forms.append((specialized, mode + '_proven_scalar_metadata', [*edits, *changes]))
for source, compiler_mode, edits in list(forms):
for plan in plans:
transferred, applied = transfer_projection_plan(source, plan)
if applied:
forms.append((transferred, compiler_mode + '_existing_plan_' + '-'.join(map(str, plan)), edits))
for source, mode, edits in forms:
fingerprint = sha(source)
if fingerprint in seen:
continue
seen.add(fingerprint)
variants.append({**record, 'label': 'joint_' + fingerprint[:16], 'code': source, 'sha256': fingerprint,
'compiler_mode': mode, 'metadata_edits': edits})
assert len(variants) <= (self.per_side + 1) ** 2 * 8
packet = dict(logical_step=logical_step, group_slot=group_slot, parent=root,
original_valid_children=records, assembly_audit=audit, observed_plans=plans, candidates=variants)
atomic(directory / 'manifest.json', packet)
accepted, formal = None, []
if variants:
best_original = min([root, *records], key=lambda r: r['score_us'])
controls = [dict(label='parent_control', code=root['code'], kind='control')]
if best_original['id'] != root['id']:
controls.append(dict(label='best_original_control', code=best_original['code'], kind='control'))
screen = self.pool.screen([*controls, *variants], key=f'search/{logical_step}/{group_slot}')
atomic(directory / 'screen.json', screen)
lookup = {r['label']: r for r in variants}
for control in controls:
result = next(r for r in screen['results'] if r['label'] == control['label'])
if not result['valid']:
raise RuntimeError('A previously valid control failed component screening')
ranked = sorted([r for r in screen['results'] if r['valid'] and r['label'] in lookup], key=lambda r: r['geometric_us'])
for choice in ranked[:self.finalists]:
candidate = lookup[choice['label']]
result = self.pool.evaluate(candidate['code'], key=f'fusion/{logical_step}/{group_slot}/{choice["label"]}')
formal.append(dict(candidate=candidate, result=result))
improving = [r for r in formal if r['result']['correctness'] > 0 and r['result']['score_us'] < best_original['score_us']]
if improving:
winner = min(improving, key=lambda r: r['result']['score_us'])
provenance = dict(logical_step=logical_step, group_slot=group_slot,
candidate_sha256=winner['candidate']['sha256'],
front_id=winner['candidate']['front_id'], tail_id=winner['candidate']['tail_id'],
result_request_id=winner['result']['request_id'],
source='script_assembly_not_policy_sample', score_us=winner['result']['score_us'])
accepted = FusionTriMulState(timestep=logical_step, construction=[], code=winner['candidate']['code'],
value=-winner['result']['score_us'], terminal_value=-winner['result']['score_us'],
step0_source=parent.step0_source, bootstrap_winner_id=parent.bootstrap_winner_id,
patch_history=list(parent.patch_history), fusion_history=[*parent.fusion_history, provenance],
component_profile=winner['result'].get('component_profile', {}))
atomic(completed, dict(logical_step=logical_step, group_slot=group_slot, candidate_count=len(variants),
original_valid_children=len(records), full_fusion_evaluations=len(formal),
raw_original_best_us=min([root['score_us'], *(r['score_us'] for r in records)]),
best_fusion_us=min([r['result']['score_us'] for r in formal if r['result']['correctness']] or [None]),
accepted_state=accepted.to_dict() if accepted else None,
training_trajectories_added=0, reward_backfills=0))
return accepted
def configure(root, ledger, **kwargs):
global RUNTIME
RUNTIME = ComponentEngine(root, ledger, **kwargs)
return RUNTIME

Xet Storage Details

Size:
16.4 kB
·
Xet hash:
7dc7e834ddc4f743a658ee4fbc8ba647aaf3f8870c8a0c2c7e1ee81e90662381

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.