Download src/psi/dlc_storage.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/dlc_storage.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/dlc_storage.py
-
curl -L -o dlc_storage.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/dlc_storage.py
8.1 kB
| """Transactional, resumable checkpoint replication. CPFS failure never kills training.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shutil | |
| import time | |
| from psi.checkpoint_lease import inference_checkpoint_lease, remove_checkpoint_unless_in_use | |
| def atomic_json(path, value): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temp = path.with_name(path.name + '.tmp') | |
| with temp.open('w') as f: | |
| json.dump(value, f, ensure_ascii=False, indent=2) | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| temp.replace(path) | |
| def digest(path): | |
| h = hashlib.sha256() | |
| with Path(path).open('rb') as f: | |
| for chunk in iter(lambda: f.read(8 << 20), b''): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def copy_verified(source, target): | |
| source, target = Path(source), Path(target) | |
| before = source.stat() | |
| if target.is_file() and target.stat().st_size == before.st_size: | |
| expected = digest(source) | |
| if digest(target) == expected: | |
| return dict(bytes=before.st_size, sha256=expected) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| temp = target.with_name(target.name + '.partial') | |
| h = hashlib.sha256() | |
| with source.open('rb') as reader, temp.open('wb') as writer: | |
| for chunk in iter(lambda: reader.read(8 << 20), b''): | |
| h.update(chunk) | |
| writer.write(chunk) | |
| writer.flush() | |
| os.fsync(writer.fileno()) | |
| after = source.stat() | |
| if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): | |
| raise OSError(f'Source changed during backup: {source}') | |
| if digest(temp) != h.hexdigest(): | |
| raise OSError(f'Backup checksum mismatch: {target}') | |
| temp.replace(target) | |
| return dict(bytes=before.st_size, sha256=h.hexdigest()) | |
| def ready(checkpoint, world): | |
| try: | |
| marker = json.loads((checkpoint / 'full_duplex_source_boundary.json').read_text()) | |
| manifest = json.loads((checkpoint / 'hfd_fused_delta_manifest.json').read_text()) | |
| required = ['model.safetensors', 'optimizer.bin', 'scheduler.bin', | |
| 'custom_checkpoint_0.pkl'] + [f'random_states_{r}.pkl' for r in range(world)] | |
| return (marker['source_group_complete'] and marker['global_step'] == int(checkpoint.name[5:]) | |
| and marker['train_infer_contract'] == manifest['contract'] | |
| and all((checkpoint / n).stat().st_size > 0 for n in required)) | |
| except (OSError, ValueError, KeyError): | |
| return False | |
| def synchronize(run, destination, world, keep=2): | |
| """Mirror public run artifacts and publish only fully verified immutable ckpts. | |
| Checkpoint leases prevent training retention deleting an in-flight source. | |
| Two committed generations are retained, only after the new one is committed. | |
| No deletion occurs outside destination/checkpoints/ckpt_<step>. | |
| """ | |
| run, destination = Path(run), Path(destination) | |
| destination.mkdir(parents=True, exist_ok=True) | |
| owner = destination / '.hfd-backup-owner.json' | |
| identity = dict(source=str(run.resolve()), format='hfd-dlc-mirror-v1') | |
| if owner.exists() and json.loads(owner.read_text()) != identity: | |
| raise ValueError('Backup destination belongs to another run') | |
| atomic_json(owner, identity) | |
| # Mutable logs are best-effort snapshots; failure leaves the last good copy. | |
| excluded = {'checkpoints', 'wandb', '.inference_snapshots', '__pycache__'} | |
| def public_files(): | |
| for folder, directories, names in os.walk(run): | |
| directories[:] = [d for d in directories if d not in excluded and not d.startswith('.')] | |
| for name in names: | |
| yield Path(folder)/name | |
| for source in public_files(): | |
| rel = source.relative_to(run) | |
| if any(part in excluded or part.startswith('.') for part in rel.parts): | |
| continue | |
| if source.is_file() and not source.is_symlink() and source.suffix not in {'.pt', '.bin', '.safetensors'}: | |
| target = destination / rel | |
| stat = source.stat() | |
| if target.exists() and target.stat().st_size == stat.st_size and target.stat().st_mtime_ns == stat.st_mtime_ns: | |
| continue | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = target.with_name(target.name + '.partial') | |
| shutil.copy2(source, temporary) | |
| temporary.replace(target) | |
| candidates = sorted((p for p in (run/'checkpoints').glob('ckpt_*') | |
| if p.name[5:].isdigit()), key=lambda p: int(p.name[5:]), reverse=True) | |
| step = None | |
| for checkpoint in candidates: | |
| if not ready(checkpoint, world): | |
| continue | |
| final = destination/'checkpoints'/checkpoint.name | |
| if (final/'backup_complete.json').exists(): | |
| step = int(checkpoint.name[5:]) | |
| break | |
| with inference_checkpoint_lease(checkpoint): | |
| if not ready(checkpoint, world): | |
| continue | |
| staging = final.with_name('.'+final.name+'.uploading') | |
| files = {} | |
| for source in sorted(checkpoint.rglob('*')): | |
| if source.is_file() and not source.is_symlink(): | |
| rel = str(source.relative_to(checkpoint)) | |
| files[rel] = copy_verified(source, staging/rel) | |
| if not files: | |
| raise OSError('Empty checkpoint') | |
| atomic_json(staging/'backup_complete.json', dict(step=int(checkpoint.name[5:]), files=files)) | |
| if final.exists(): | |
| raise FileExistsError(f'Uncommitted destination exists: {final}') | |
| staging.rename(final) | |
| step = int(checkpoint.name[5:]) | |
| break | |
| if step is not None: | |
| atomic_json(destination/'latest_backup.json', dict(step=step, checkpoint=f'checkpoints/ckpt_{step}', updated=time.time())) | |
| committed = sorted((p for p in (destination/'checkpoints').glob('ckpt_*') | |
| if p.name[5:].isdigit() and (p/'backup_complete.json').is_file()), | |
| key=lambda p: int(p.name[5:]), reverse=True) | |
| for old in committed[max(2, keep):]: | |
| remove_checkpoint_unless_in_use(old) | |
| # Only our owned, unpublished staging generations older than the now | |
| # verified checkpoint can be discarded; never touch another run. | |
| for old in (destination/'checkpoints').glob('.ckpt_*.uploading'): | |
| suffix = old.name[len('.ckpt_'):-len('.uploading')] | |
| if suffix.isdigit() and int(suffix) <= step and old.is_dir() and not old.is_symlink(): | |
| shutil.rmtree(old) | |
| return step | |
| def restore(backup, target, world): | |
| """Restore one committed full state, validating every recorded checksum.""" | |
| backup, target = Path(backup), Path(target) | |
| pointer = json.loads((backup/'latest_backup.json').read_text()) | |
| name = f"ckpt_{int(pointer['step'])}" | |
| source = backup/'checkpoints'/name | |
| receipt = json.loads((source/'backup_complete.json').read_text()) | |
| rng_ranks = {p.name for p in source.glob('random_states_*.pkl')} | |
| if rng_ranks != {f'random_states_{r}.pkl' for r in range(world)} or not ready(source, world): | |
| raise ValueError('Backup is incomplete or has a different rank count') | |
| checkpoint = target/'checkpoints'/name | |
| for rel, expected in receipt['files'].items(): | |
| if Path(rel).is_absolute() or '..' in Path(rel).parts: | |
| raise ValueError('Invalid backup member') | |
| actual = copy_verified(source/rel, checkpoint/rel) | |
| if actual != expected: | |
| raise OSError(f'Backup checksum mismatch: {rel}') | |
| for filename in ('run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json'): | |
| copy_verified(backup/filename, target/filename) | |
| shutil.copytree(backup/'control_tokenizer', target/'control_tokenizer', dirs_exist_ok=True) | |
| if not ready(checkpoint, world): | |
| raise OSError('Restored checkpoint is incomplete') | |
| return checkpoint | |