zefancainlp's picture
download
raw
8.43 kB
"""Three new original MaxRL batches. Exact-token audit and node-only exports."""
from __future__ import annotations
import argparse
import asyncio
import fcntl
import logging
import math
import os
from pathlib import Path
import sys
import time
def config_for(root):
import integration
from common import read
from ttt_discover.rl.train import Config
from ttt_discover.tinker_utils.dataset_builder import DatasetConfig
plan = read(Path(root)/'control/plan.json')
data = DatasetConfig(problem_type='trimul',env_type=integration.Env,batch_size=8,
model_name_for_tokenizer=plan['tokenizer'],renderer_name='gpt_oss_high_reasoning',group_size=64,
num_cpus_per_task=1,eval_timeout=530,timeout=None,log_path=str(Path(root)/'run'),parent_selection_policy='puct')
return Config(env_type=integration.Env,problem_type='trimul',dataset_builder=integration.Builder(config=data),
backend='tinker',model_name='openai/gpt-oss-120b',local_model_path=plan['tokenizer'],
local_gpu_context_window=32768,lora_rank=32,num_epochs=3,learning_rate=4e-5,
temperature=1.,kl_penalty_coef=.1,adv_estimator='entropic_adaptive_beta',adv_estimator_kl_nats=math.log(2.),
loss_fn='importance_sampling',num_substeps=1,save_every=1,remove_constant_reward_groups=True,
phase1_max_tokens=26000,log_path=str(Path(root)/'run'),wandb_project=None,
wandb_name='kernel-private-entry-split-step0-api3',trace_credit_weight=0.,factorized_action_credit=False,
breakthrough_replay_enabled=False,descendant_credit_enabled=False,bellman_relabel_enabled=False,
hindsight_shortcut_enabled=False,credit_probe_steps=(),load_checkpoint_path=None)
async def run(root):
import api_transport
import integration
import ttt_discover.rl.train as native
from common import NEW_STEPS,atomic,install_api_guard,read,sha
from local_storage import LocalStorage,sha as file_sha
from ttt_discover.tinker_utils import ml_log
from transformers import AutoTokenizer
import shutil
runtime = integration.configure(root)
cfg = config_for(root)
tokenizer = AutoTokenizer.from_pretrained(cfg.local_model_path,use_fast=True,local_files_only=True)
dataset = await cfg.dataset_builder()
dataset.sampler.flush(0)
health = runtime.judge.health()
states = health['states']
state = states[0]
if any(s['pids'] or s['memory_mib'] >= 100 or s['temperature'] >= 75 for s in states):
raise RuntimeError('Dedicated timing GPU unavailable; no paid actor created')
if shutil.disk_usage(root).free < 30_000_000_000:
raise RuntimeError('Insufficient node storage; no paid actor created')
calibration = read(root/'control/gpu_preflight.json')
if not calibration['success'] or calibration['gpu_uuids'] != [s['uuid'] for s in states]:
raise RuntimeError('No calibration for the exact timing GPU; no paid actor created')
install_api_guard(api_transport,root)
service = api_transport.make_service(root,allow_paid=True)
runtime.ledger = service.ledger
atomic(root/'control/actor_create_intent.json',dict(model=cfg.model_name,rank=32,time=time.time(),
fresh_retry_on_unknown_outcome_forbidden=True))
actor = await service.create_lora_training_client_async(cfg.model_name,rank=32)
atomic(root/'control/actor.json',dict(actor_id=str(actor.client.model_id),pid=os.getpid(),time=time.time(),
maximum_new_steps=3,initial_policy='fresh base LoRA; no historical weights or Adam imported',
persistent_cloud_checkpoints=False,optimizer_live_only=True))
store = LocalStorage(root,service,actor)
store.samplers[0] = await actor.save_weights_and_get_sampling_client_async()
directory = root/'node_checkpoints/untrained_base'
atomic(directory/'base_policy.json',dict(base_model=cfg.model_name,lora_rank=32,untrained=True,optimizer_updates=0))
store.commit(0,directory,[dict(path='base_policy.json',bytes=(directory/'base_policy.json').stat().st_size,
sha256=file_sha(directory/'base_policy.json'))])
native.save_checkpoint_and_get_sampling_client = store.boundary
async def forbidden(*args,**kwargs):
raise api_transport.RequestSafetyError('Persistent cloud checkpoints prohibited; verified node exports only')
api_transport.TrainingProxy.save_state_async = forbidden
api_transport.TrainingProxy.save_weights_for_sampler_async = forbidden
native.save_checkpoint_async = forbidden
original_rollout = native.do_group_rollout
async def exact_group(*args,**kwargs):
import pickle
result = await original_rollout(*args,**kwargs)
builder = kwargs.get('group_builder') or next((v for v in args if isinstance(v,integration.Group)),None)
if builder is None:
builder = next((v for v in kwargs.values() if isinstance(v,integration.Group)),None)
if builder is None:
raise RuntimeError('Cannot identify exact native rollout group for audit')
path = root/'run/exact_native_groups'/f's{builder.logical_step:02d}'/f'g{builder.group_slot:02d}.pkl'
path.parent.mkdir(parents=True,exist_ok=True)
try:
payload = pickle.dumps(result,protocol=5)
except Exception as error:
atomic(path.with_suffix('.unsupported.json'),dict(complete=False,error=type(error).__name__,
exact_tokens_logprobs_separately_cached=True,resume_not_claimed=True))
else:
with path.open('xb') as handle:
handle.write(payload)
atomic(path.with_suffix('.json'),dict(complete=True,sha256=sha(payload),bytes=len(payload),reconstructed=False))
return result
native.do_group_rollout = exact_group
logger = ml_log.setup_logging(log_dir=cfg.log_path,wandb_project=None,config=cfg,wandb_name=cfg.wandb_name)
atomic(root/'run/resolved_config.json',read(root/'control/scientific_config.json'))
atomic(root/'control/status.json',dict(status='running',new_steps=3,Step0_reused=True,time=time.time()))
try:
await native.do_sync_training(start_batch=0,end_batch=NEW_STEPS,num_batches=NEW_STEPS,cfg=cfg,
training_client=actor,service_client=service,dataset=dataset,ml_logger=logger,tokenizer=tokenizer)
finally:
logger.close()
rows = [__import__('json').loads(s) for s in service.ledger.log.read_text().splitlines()]
updates = [r for r in rows if r.get('event')=='completed' and r.get('operation')=='optim_step']
samples = [r for r in rows if r.get('event')=='completed' and r.get('operation')=='sample']
slots = {tuple(r['slot']) for r in samples}
expected = {(s,g,e) for s in range(1,4) for g in range(8) for e in range(64)}
if store.latest['batch']!=3 or len(updates)!=3 or slots!=expected:
raise RuntimeError('Final three-step/update/exact-episode boundary mismatch')
atomic(root/'run/verification.json',dict(success=True,new_updates=3,new_original_episodes=1536,
Step0_resampled=False,Step0_updated=False,historical_optimizer_restored=False,
persistent_cloud_checkpoints=False,optimizer_on_node=False,final_boundary=store.latest))
atomic(root/'control/status.json',dict(status='complete',new_steps=3,time=time.time()))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--root',type=Path,required=True)
parser.add_argument('--execute',action='store_true',required=True)
args = parser.parse_args()
root = args.root.resolve()
sys.path[:0] = [str(root/'code'),str(root/'source')]
from common import atomic,verify_ready
verify_ready(root)
os.environ.update(TTT_ALLOW_PAID_KERNEL_API10='1',TTT_ALLOW_KERNEL_PRIVATE_API3='1',
TTT_WANDB_TABLES='0',WANDB_MODE='disabled',PYTHONUNBUFFERED='1',CUDA_VISIBLE_DEVICES='')
os.environ.pop('TTT_TRIMUL_GUARD_POLICY',None)
with (root/'control/active.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
logging.basicConfig(level=logging.INFO,filename=root/'run/train.log')
try:
asyncio.run(run(root))
except BaseException as error:
atomic(root/'control/status.json',dict(status='needs_attention',error=repr(error),time=time.time(),
automatic_fresh_restart_forbidden=True))
raise
if __name__ == '__main__':
main()

Xet Storage Details

Size:
8.43 kB
·
Xet hash:
ea4e5caa25a9d8e81cbcc37862555694f0901dc916f45de5553017151ab3b958

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.