Buckets:
| """Execution-only overlay; do not edit the sealed method or restart policy.""" | |
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import functools | |
| import hashlib | |
| import json | |
| import logging | |
| from pathlib import Path | |
| import sys | |
| import psutil | |
| class RecheckedSemaphore: | |
| """Stop queued requests BEFORE charging, while dispatched ones finish.""" | |
| def __init__(self, ledger, capacity): | |
| self.ledger = ledger | |
| self.semaphore = asyncio.Semaphore(capacity) | |
| async def __aenter__(self): | |
| await self.semaphore.acquire() | |
| try: | |
| self.ledger.require_paid_authorization() | |
| except BaseException: | |
| self.semaphore.release() | |
| raise | |
| return self | |
| async def __aexit__(self, *args): | |
| self.semaphore.release() | |
| class DrainGuard: | |
| def __init__(self): | |
| self.ledgers = [] | |
| self.inflight = set() | |
| def track(self, function): | |
| async def tracked(*args, **kwargs): | |
| done = asyncio.get_running_loop().create_future() | |
| self.inflight.add(done) | |
| try: | |
| return await function(*args, **kwargs) | |
| finally: | |
| done.set_result(None) | |
| self.inflight.discard(done) | |
| return tracked | |
| def protect(self, function): | |
| async def protected(*args, **kwargs): | |
| try: | |
| return await function(*args, **kwargs) | |
| except BaseException: | |
| for ledger in self.ledgers: | |
| ledger.enabled = False | |
| # On an evaluator exception, asyncio.gather has not cancelled | |
| # sibling episodes. Drain their already-dispatched responses | |
| # into the original exact-token cache before closing the loop. | |
| while self.inflight: | |
| await asyncio.gather(*list(self.inflight), return_exceptions=True) | |
| raise | |
| return protected | |
| def install(root=None): | |
| import gpu_pool | |
| import api_transport | |
| import ttt_discover.rl.train as native | |
| original_owned = gpu_pool.JudgePool.owned_pids | |
| original_gpu_state = gpu_pool.gpu_state | |
| original_request = gpu_pool.JudgePool.request | |
| known_owned = {} | |
| def safe_owned(container): | |
| try: | |
| observed = original_owned(container) | |
| except psutil.NoSuchProcess: | |
| observed = set() | |
| prior = known_owned.setdefault(container, {}) | |
| for pid in observed: | |
| try: | |
| prior[pid] = psutil.Process(pid).create_time() | |
| except psutil.NoSuchProcess: | |
| pass | |
| retained = set() | |
| for pid, created in prior.items(): | |
| try: | |
| if psutil.Process(pid).create_time() == created: | |
| retained.add(pid) | |
| except psutil.NoSuchProcess: | |
| pass | |
| return observed | retained | |
| def fresh_gpu_state(index): | |
| state = original_gpu_state(index) | |
| live = [] | |
| for pid in state['pids']: | |
| try: | |
| if psutil.Process(pid).status() != psutil.STATUS_ZOMBIE: | |
| live.append(pid) | |
| except psutil.NoSuchProcess: | |
| pass | |
| state['pids'] = live | |
| return state | |
| def retry_interrupted_local_judge(self, payload, *, key, timeout=530): | |
| for attempt in range(3): | |
| try: | |
| return original_request(self, payload, key=key, timeout=timeout) | |
| except gpu_pool.JudgeInfrastructureError as error: | |
| if 'interrupted by occupancy/temperature' not in str(error) or attempt == 2: | |
| raise | |
| # Repeat the SAME local evaluation after its GPU lease is | |
| # released, never the LLM rollout. Candidate timeouts, wrong | |
| # answers and compilation failures are NOT retried. | |
| logging.warning('Local judge resource interruption: retry same key %s, local attempt %s', key, attempt + 2) | |
| gpu_pool.JudgePool.owned_pids = staticmethod(safe_owned) | |
| gpu_pool.gpu_state = fresh_gpu_state | |
| gpu_pool.JudgePool.request = retry_interrupted_local_judge | |
| if root is not None: | |
| import ttt_discover.tinker_utils.sampler as sampler_module | |
| from ttt_discover.tinker_utils.misc_utils import get_last_checkpoint | |
| original_factory = sampler_module.create_sampler | |
| def resume_sampler(*, log_path, **kwargs): | |
| if Path(log_path).resolve() == (Path(root) / 'run').resolve() and kwargs.get('resume_step') is None: | |
| checkpoint = get_last_checkpoint(log_path) | |
| if checkpoint is not None: | |
| step = int(checkpoint['batch']) | |
| boundary = json.loads((Path(log_path) / 'checkpoint_boundaries' / f'{step:06d}.json').read_text()) | |
| state_file = Path(boundary['sampler_path_local']) | |
| if hashlib.sha256(state_file.read_bytes()).hexdigest() != boundary['sampler_sha256']: | |
| raise RuntimeError('Do not rebuild or overwrite an unmatched sampler boundary') | |
| kwargs['resume_step'] = step # Zero is a real saved boundary, not fresh initialization. | |
| return original_factory(log_path=log_path, **kwargs) | |
| sampler_module.create_sampler = resume_sampler | |
| guard = DrainGuard() | |
| original_make = api_transport.make_service | |
| def service(*args, **kwargs): | |
| proxy = original_make(*args, **kwargs) | |
| guard.ledgers.append(proxy.ledger) | |
| proxy.ledger._semaphore = RecheckedSemaphore(proxy.ledger, proxy.ledger.sampling_concurrency) | |
| return proxy | |
| api_transport.make_service = service | |
| api_transport.SamplingProxy.sample_async = guard.track(api_transport.SamplingProxy.sample_async) | |
| api_transport.SamplingProxy.compute_logprobs_async = guard.track(api_transport.SamplingProxy.compute_logprobs_async) | |
| native.main = guard.protect(native.main) | |
| return guard | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--root', required=True, type=Path) | |
| root = parser.parse_args().root.resolve() | |
| sys.path[:0] = [str(root / 'code'), str(root / 'source')] | |
| import train_api | |
| train_api.assert_frozen(root) | |
| proof = json.loads((root / 'control/recovery/first_gpu_exit_race/retrieval_summary.json').read_text()) | |
| if not proof['success'] or proof['new_paid_model_calls'] != 0: | |
| raise RuntimeError('Reconcile the interrupted paid responses before resuming') | |
| install(root) | |
| sys.argv = [str(root / 'code/train_api.py'), '--root', str(root), '--execute'] | |
| train_api.main() | |
Xet Storage Details
- Size:
- 6.73 kB
- Xet hash:
- d584ee918d5ca8924b7acf93e4c261e9debf6ec581abecf97ea5ef4c47adcb39
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.