zefancainlp's picture
download
raw
6.73 kB
"""Execution-only overlay; do not edit the sealed method or restart policy."""
from __future__ import annotations
import argparse
import asyncio
import functools
import hashlib
import json
import logging
from pathlib import Path
import sys
import psutil
class RecheckedSemaphore:
"""Stop queued requests BEFORE charging, while dispatched ones finish."""
def __init__(self, ledger, capacity):
self.ledger = ledger
self.semaphore = asyncio.Semaphore(capacity)
async def __aenter__(self):
await self.semaphore.acquire()
try:
self.ledger.require_paid_authorization()
except BaseException:
self.semaphore.release()
raise
return self
async def __aexit__(self, *args):
self.semaphore.release()
class DrainGuard:
def __init__(self):
self.ledgers = []
self.inflight = set()
def track(self, function):
@functools.wraps(function)
async def tracked(*args, **kwargs):
done = asyncio.get_running_loop().create_future()
self.inflight.add(done)
try:
return await function(*args, **kwargs)
finally:
done.set_result(None)
self.inflight.discard(done)
return tracked
def protect(self, function):
@functools.wraps(function)
async def protected(*args, **kwargs):
try:
return await function(*args, **kwargs)
except BaseException:
for ledger in self.ledgers:
ledger.enabled = False
# On an evaluator exception, asyncio.gather has not cancelled
# sibling episodes. Drain their already-dispatched responses
# into the original exact-token cache before closing the loop.
while self.inflight:
await asyncio.gather(*list(self.inflight), return_exceptions=True)
raise
return protected
def install(root=None):
import gpu_pool
import api_transport
import ttt_discover.rl.train as native
original_owned = gpu_pool.JudgePool.owned_pids
original_gpu_state = gpu_pool.gpu_state
original_request = gpu_pool.JudgePool.request
known_owned = {}
def safe_owned(container):
try:
observed = original_owned(container)
except psutil.NoSuchProcess:
observed = set()
prior = known_owned.setdefault(container, {})
for pid in observed:
try:
prior[pid] = psutil.Process(pid).create_time()
except psutil.NoSuchProcess:
pass
retained = set()
for pid, created in prior.items():
try:
if psutil.Process(pid).create_time() == created:
retained.add(pid)
except psutil.NoSuchProcess:
pass
return observed | retained
def fresh_gpu_state(index):
state = original_gpu_state(index)
live = []
for pid in state['pids']:
try:
if psutil.Process(pid).status() != psutil.STATUS_ZOMBIE:
live.append(pid)
except psutil.NoSuchProcess:
pass
state['pids'] = live
return state
def retry_interrupted_local_judge(self, payload, *, key, timeout=530):
for attempt in range(3):
try:
return original_request(self, payload, key=key, timeout=timeout)
except gpu_pool.JudgeInfrastructureError as error:
if 'interrupted by occupancy/temperature' not in str(error) or attempt == 2:
raise
# Repeat the SAME local evaluation after its GPU lease is
# released, never the LLM rollout. Candidate timeouts, wrong
# answers and compilation failures are NOT retried.
logging.warning('Local judge resource interruption: retry same key %s, local attempt %s', key, attempt + 2)
gpu_pool.JudgePool.owned_pids = staticmethod(safe_owned)
gpu_pool.gpu_state = fresh_gpu_state
gpu_pool.JudgePool.request = retry_interrupted_local_judge
if root is not None:
import ttt_discover.tinker_utils.sampler as sampler_module
from ttt_discover.tinker_utils.misc_utils import get_last_checkpoint
original_factory = sampler_module.create_sampler
def resume_sampler(*, log_path, **kwargs):
if Path(log_path).resolve() == (Path(root) / 'run').resolve() and kwargs.get('resume_step') is None:
checkpoint = get_last_checkpoint(log_path)
if checkpoint is not None:
step = int(checkpoint['batch'])
boundary = json.loads((Path(log_path) / 'checkpoint_boundaries' / f'{step:06d}.json').read_text())
state_file = Path(boundary['sampler_path_local'])
if hashlib.sha256(state_file.read_bytes()).hexdigest() != boundary['sampler_sha256']:
raise RuntimeError('Do not rebuild or overwrite an unmatched sampler boundary')
kwargs['resume_step'] = step # Zero is a real saved boundary, not fresh initialization.
return original_factory(log_path=log_path, **kwargs)
sampler_module.create_sampler = resume_sampler
guard = DrainGuard()
original_make = api_transport.make_service
def service(*args, **kwargs):
proxy = original_make(*args, **kwargs)
guard.ledgers.append(proxy.ledger)
proxy.ledger._semaphore = RecheckedSemaphore(proxy.ledger, proxy.ledger.sampling_concurrency)
return proxy
api_transport.make_service = service
api_transport.SamplingProxy.sample_async = guard.track(api_transport.SamplingProxy.sample_async)
api_transport.SamplingProxy.compute_logprobs_async = guard.track(api_transport.SamplingProxy.compute_logprobs_async)
native.main = guard.protect(native.main)
return guard
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--root', required=True, type=Path)
root = parser.parse_args().root.resolve()
sys.path[:0] = [str(root / 'code'), str(root / 'source')]
import train_api
train_api.assert_frozen(root)
proof = json.loads((root / 'control/recovery/first_gpu_exit_race/retrieval_summary.json').read_text())
if not proof['success'] or proof['new_paid_model_calls'] != 0:
raise RuntimeError('Reconcile the interrupted paid responses before resuming')
install(root)
sys.argv = [str(root / 'code/train_api.py'), '--root', str(root), '--execute']
train_api.main()

Xet Storage Details

Size:
6.73 kB
·
Xet hash:
d584ee918d5ca8924b7acf93e4c261e9debf6ec581abecf97ea5ef4c47adcb39

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.