"""Transactional, resumable checkpoint replication. CPFS failure never kills training.""" from __future__ import annotations import hashlib import json import os from pathlib import Path import shutil import time from psi.checkpoint_lease import inference_checkpoint_lease, remove_checkpoint_unless_in_use def atomic_json(path, value): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_name(path.name + '.tmp') with temp.open('w') as f: json.dump(value, f, ensure_ascii=False, indent=2) f.flush() os.fsync(f.fileno()) temp.replace(path) def digest(path): h = hashlib.sha256() with Path(path).open('rb') as f: for chunk in iter(lambda: f.read(8 << 20), b''): h.update(chunk) return h.hexdigest() def copy_verified(source, target): source, target = Path(source), Path(target) before = source.stat() if target.is_file() and target.stat().st_size == before.st_size: expected = digest(source) if digest(target) == expected: return dict(bytes=before.st_size, sha256=expected) target.parent.mkdir(parents=True, exist_ok=True) temp = target.with_name(target.name + '.partial') h = hashlib.sha256() with source.open('rb') as reader, temp.open('wb') as writer: for chunk in iter(lambda: reader.read(8 << 20), b''): h.update(chunk) writer.write(chunk) writer.flush() os.fsync(writer.fileno()) after = source.stat() if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): raise OSError(f'Source changed during backup: {source}') if digest(temp) != h.hexdigest(): raise OSError(f'Backup checksum mismatch: {target}') temp.replace(target) return dict(bytes=before.st_size, sha256=h.hexdigest()) def ready(checkpoint, world): try: marker = json.loads((checkpoint / 'full_duplex_source_boundary.json').read_text()) manifest = json.loads((checkpoint / 'hfd_fused_delta_manifest.json').read_text()) required = ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl'] + [f'random_states_{r}.pkl' for r in range(world)] return (marker['source_group_complete'] and marker['global_step'] == int(checkpoint.name[5:]) and marker['train_infer_contract'] == manifest['contract'] and all((checkpoint / n).stat().st_size > 0 for n in required)) except (OSError, ValueError, KeyError): return False def synchronize(run, destination, world, keep=2): """Mirror public run artifacts and publish only fully verified immutable ckpts. Checkpoint leases prevent training retention deleting an in-flight source. Two committed generations are retained, only after the new one is committed. No deletion occurs outside destination/checkpoints/ckpt_. """ run, destination = Path(run), Path(destination) destination.mkdir(parents=True, exist_ok=True) owner = destination / '.hfd-backup-owner.json' identity = dict(source=str(run.resolve()), format='hfd-dlc-mirror-v1') if owner.exists() and json.loads(owner.read_text()) != identity: raise ValueError('Backup destination belongs to another run') atomic_json(owner, identity) # Mutable logs are best-effort snapshots; failure leaves the last good copy. excluded = {'checkpoints', 'wandb', '.inference_snapshots', '__pycache__'} def public_files(): for folder, directories, names in os.walk(run): directories[:] = [d for d in directories if d not in excluded and not d.startswith('.')] for name in names: yield Path(folder)/name for source in public_files(): rel = source.relative_to(run) if any(part in excluded or part.startswith('.') for part in rel.parts): continue if source.is_file() and not source.is_symlink() and source.suffix not in {'.pt', '.bin', '.safetensors'}: target = destination / rel stat = source.stat() if target.exists() and target.stat().st_size == stat.st_size and target.stat().st_mtime_ns == stat.st_mtime_ns: continue target.parent.mkdir(parents=True, exist_ok=True) temporary = target.with_name(target.name + '.partial') shutil.copy2(source, temporary) temporary.replace(target) candidates = sorted((p for p in (run/'checkpoints').glob('ckpt_*') if p.name[5:].isdigit()), key=lambda p: int(p.name[5:]), reverse=True) step = None for checkpoint in candidates: if not ready(checkpoint, world): continue final = destination/'checkpoints'/checkpoint.name if (final/'backup_complete.json').exists(): step = int(checkpoint.name[5:]) break with inference_checkpoint_lease(checkpoint): if not ready(checkpoint, world): continue staging = final.with_name('.'+final.name+'.uploading') files = {} for source in sorted(checkpoint.rglob('*')): if source.is_file() and not source.is_symlink(): rel = str(source.relative_to(checkpoint)) files[rel] = copy_verified(source, staging/rel) if not files: raise OSError('Empty checkpoint') atomic_json(staging/'backup_complete.json', dict(step=int(checkpoint.name[5:]), files=files)) if final.exists(): raise FileExistsError(f'Uncommitted destination exists: {final}') staging.rename(final) step = int(checkpoint.name[5:]) break if step is not None: atomic_json(destination/'latest_backup.json', dict(step=step, checkpoint=f'checkpoints/ckpt_{step}', updated=time.time())) committed = sorted((p for p in (destination/'checkpoints').glob('ckpt_*') if p.name[5:].isdigit() and (p/'backup_complete.json').is_file()), key=lambda p: int(p.name[5:]), reverse=True) for old in committed[max(2, keep):]: remove_checkpoint_unless_in_use(old) # Only our owned, unpublished staging generations older than the now # verified checkpoint can be discarded; never touch another run. for old in (destination/'checkpoints').glob('.ckpt_*.uploading'): suffix = old.name[len('.ckpt_'):-len('.uploading')] if suffix.isdigit() and int(suffix) <= step and old.is_dir() and not old.is_symlink(): shutil.rmtree(old) return step def restore(backup, target, world): """Restore one committed full state, validating every recorded checksum.""" backup, target = Path(backup), Path(target) pointer = json.loads((backup/'latest_backup.json').read_text()) name = f"ckpt_{int(pointer['step'])}" source = backup/'checkpoints'/name receipt = json.loads((source/'backup_complete.json').read_text()) rng_ranks = {p.name for p in source.glob('random_states_*.pkl')} if rng_ranks != {f'random_states_{r}.pkl' for r in range(world)} or not ready(source, world): raise ValueError('Backup is incomplete or has a different rank count') checkpoint = target/'checkpoints'/name for rel, expected in receipt['files'].items(): if Path(rel).is_absolute() or '..' in Path(rel).parts: raise ValueError('Invalid backup member') actual = copy_verified(source/rel, checkpoint/rel) if actual != expected: raise OSError(f'Backup checksum mismatch: {rel}') for filename in ('run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json'): copy_verified(backup/filename, target/filename) shutil.copytree(backup/'control_tokenizer', target/'control_tokenizer', dirs_exist_ok=True) if not ready(checkpoint, world): raise OSError('Restored checkpoint is incomplete') return checkpoint