zefancainlp's picture
download
raw
8.84 kB
"""Single authorized 10-logical-step run; Step 0 is imported, Steps 1..9 learn.
The frozen framework owns sampling, MaxRL, KL, loss, Adam and PUCT. This entry
only installs the new environment/group search, exact API cache, and durable
checkpoint adapters. No extra paid control/smoke experiment is launched.
"""
from __future__ import annotations
import argparse
import asyncio
import fcntl
import hashlib
import json
import logging
import math
import os
from pathlib import Path
import sys
import time
def assert_frozen(root):
manifest = json.loads((root / 'control/ready.json').read_text())
if not manifest.get('ready') or manifest.get('paid_requests_during_preflight') != 0:
raise RuntimeError('No complete zero-paid-computation preflight receipt')
for relative, expected in manifest['files'].items():
if hashlib.sha256((root / relative).read_bytes()).hexdigest() != expected:
raise RuntimeError('Prepared source changed: ' + relative)
return manifest
def assert_safe_resume(root, checkpoint_batch):
path = root / 'run/api_audit/api_requests.jsonl'
if not path.exists():
return
rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
uncommitted_training = [r for r in rows if r.get('operation') in ['forward_backward', 'optim_step']
and r['logical_step'] > checkpoint_batch]
if uncommitted_training:
raise RuntimeError('An uncommitted paid training operation exists. Reconcile its server/checkpoint state before any resubmission.')
def build_config(root):
from ttt_discover.rl.train import Config
from ttt_discover.tinker_utils.dataset_builder import DatasetConfig
from integration import ComponentDatasetBuilder, ComponentTriMulEnv
tokenizer = '/mnt/localssd/models/gpt-oss-120b'
data = DatasetConfig(problem_type='trimul', env_type=ComponentTriMulEnv,
batch_size=8, model_name_for_tokenizer=tokenizer,
renderer_name='gpt_oss_high_reasoning', group_size=64,
num_cpus_per_task=1, eval_timeout=530, log_path=str(root / 'run'),
timeout=None, parent_selection_policy='puct')
return Config(env_type=ComponentTriMulEnv, problem_type='trimul',
dataset_builder=ComponentDatasetBuilder(config=data), backend='tinker',
model_name='openai/gpt-oss-120b', local_model_path=tokenizer,
local_gpu_context_window=32768, lora_rank=32, num_epochs=9,
learning_rate=4e-5, temperature=1., kl_penalty_coef=.1,
adv_estimator='entropic_adaptive_beta', adv_estimator_kl_nats=math.log(2.),
loss_fn='importance_sampling', num_substeps=1, save_every=1,
remove_constant_reward_groups=True, phase1_max_tokens=26000,
log_path=str(root / 'run'), wandb_project=None, wandb_name='kernel-component-api10',
trace_credit_weight=0., factorized_action_credit=False,
breakthrough_replay_enabled=False, descendant_credit_enabled=False,
bellman_relabel_enabled=False, hindsight_shortcut_enabled=False,
credit_probe_steps=(), load_checkpoint_path=None)
def install_durable_checkpoints(train_module):
from ttt_discover.tinker_utils.misc_utils import get_last_checkpoint, save_checkpoint_async
async def durable(training_client, i_batch, log_path, save_every, start_batch=0):
if save_every != 1:
raise RuntimeError('One complete API/optimizer checkpoint per committed batch is required')
prior = get_last_checkpoint(log_path)
if prior and int(prior['batch']) == i_batch and prior.get('sampler_path'):
return training_client.create_sampling_client(prior['sampler_path']), {'checkpoint/reused_complete_boundary': 1}
paths = await save_checkpoint_async(training_client, f'{i_batch:06d}', log_path,
loop_state={'batch': i_batch}, kind='both')
from api_transport import atomic
log = Path(log_path)
sampler = log / f'puct_sampler_step_{i_batch:06d}.json'
if not sampler.exists():
raise RuntimeError('A checkpoint cannot be complete without its exact sampler boundary')
atomic(log / 'checkpoint_boundaries' / f'{i_batch:06d}.json',
dict(batch=i_batch, policy_boundary=i_batch, **paths,
sampler_path_local=str(sampler), sampler_sha256=hashlib.sha256(sampler.read_bytes()).hexdigest(),
optimizer_included=True))
summary = log / 'step_summaries' / f'{i_batch:02d}.json'
if summary.exists():
row = json.loads(summary.read_text())
row.update(status='complete_checkpoint', checkpoint_boundary=i_batch)
atomic(summary, row)
return training_client.create_sampling_client(paths['sampler_path']), {}
train_module.save_checkpoint_and_get_sampling_client = durable
async def run(root):
from api_transport import atomic, make_service
import integration
import ttt_discover.rl.train as train_module
from ttt_discover.tinker_utils.misc_utils import get_last_checkpoint
checkpoint = get_last_checkpoint(str(root / 'run'))
batch = int(checkpoint['batch']) if checkpoint else 0
assert_safe_resume(root, batch)
if batch >= 9:
print('ALREADY_COMPLETE_NO_API_COMPUTATION', flush=True)
return
service = make_service(root, allow_paid=True)
integration.configure(root, service.ledger)
# Only the constructor used by this frozen module is replaced; types and RL
# math functions remain the original imported SDK/framework implementations.
train_module.tinker.ServiceClient = lambda **kwargs: service
install_durable_checkpoints(train_module)
cfg = build_config(root)
config_receipt = dict(model=cfg.model_name, backend=cfg.backend, lora_rank=cfg.lora_rank,
logical_steps=10, imported_step0_updates=0, fresh_batches=9,
groups_per_batch=8, group_size=64, fresh_episodes=4608,
learning_rate=cfg.learning_rate, kl_penalty_coef=cfg.kl_penalty_coef,
advantage=cfg.adv_estimator, loss=cfg.loss_fn, num_substeps=cfg.num_substeps,
phase1_total_context=cfg.phase1_max_tokens, context_window=cfg.local_gpu_context_window,
per_candidate_execution_seconds=530, source_library='parent_and_current_64_siblings_only',
training_data='original_512_only_with_original_own_rewards',
fusion_reward_backfills=False, synthetic_training_trajectories=False,
checkpoint_ttl_seconds=7 * 86400, dollar_cap=None,
user_budget_authorization='User explicitly authorized no dollar cap; no extra rollouts or paid smoke.',
steps_on_resume_remaining=9 - batch)
atomic(root / 'run/resolved_config.json', config_receipt)
atomic(root / 'control/status.json', dict(status='starting_api_run', started_at=time.time(), resume_batch=batch))
await train_module.main(cfg)
final = get_last_checkpoint(str(root / 'run'))
if not final or int(final['batch']) != 9:
raise RuntimeError('Training returned without the final nine-update checkpoint')
atomic(root / 'control/status.json', dict(status='complete', completed_at=time.time(), final_checkpoint=final,
imported_step0=True, fresh_batches_completed=9, logical_steps=10))
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--root', type=Path, required=True)
ap.add_argument('--execute', action='store_true', required=True)
args = ap.parse_args()
root = args.root.resolve()
assert_frozen(root)
sys.path[:0] = [str(root / 'code'), str(root / 'source')]
os.chdir(root / 'source')
os.environ.update(TTT_ALLOW_PAID_KERNEL_API10='1', TTT_WANDB_TABLES='0', WANDB_MODE='disabled', PYTHONUNBUFFERED='1')
logging.basicConfig(level=logging.INFO, filename=root / 'run/train.log', filemode='a')
with (root / 'control/active.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
asyncio.run(run(root))
except BaseException as error:
from api_transport import atomic
atomic(root / 'control/status.json', dict(status='needs_attention', time=time.time(), error=type(error).__name__ + ': ' + str(error)[:2000]))
logging.exception('API run stopped; never auto-submit an uncertain paid operation again')
raise
if __name__ == '__main__':
main()

Xet Storage Details

Size:
8.84 kB
·
Xet hash:
6e7ee146769e6d04619a2182433c9f97e4ac1ee570984b383f60d331937e7772

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.