code-HFD / src /psi /dlc_storage.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
8.1 kB
"""Transactional, resumable checkpoint replication. CPFS failure never kills training."""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import shutil
import time
from psi.checkpoint_lease import inference_checkpoint_lease, remove_checkpoint_unless_in_use
def atomic_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_name(path.name + '.tmp')
with temp.open('w') as f:
json.dump(value, f, ensure_ascii=False, indent=2)
f.flush()
os.fsync(f.fileno())
temp.replace(path)
def digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as f:
for chunk in iter(lambda: f.read(8 << 20), b''):
h.update(chunk)
return h.hexdigest()
def copy_verified(source, target):
source, target = Path(source), Path(target)
before = source.stat()
if target.is_file() and target.stat().st_size == before.st_size:
expected = digest(source)
if digest(target) == expected:
return dict(bytes=before.st_size, sha256=expected)
target.parent.mkdir(parents=True, exist_ok=True)
temp = target.with_name(target.name + '.partial')
h = hashlib.sha256()
with source.open('rb') as reader, temp.open('wb') as writer:
for chunk in iter(lambda: reader.read(8 << 20), b''):
h.update(chunk)
writer.write(chunk)
writer.flush()
os.fsync(writer.fileno())
after = source.stat()
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
raise OSError(f'Source changed during backup: {source}')
if digest(temp) != h.hexdigest():
raise OSError(f'Backup checksum mismatch: {target}')
temp.replace(target)
return dict(bytes=before.st_size, sha256=h.hexdigest())
def ready(checkpoint, world):
try:
marker = json.loads((checkpoint / 'full_duplex_source_boundary.json').read_text())
manifest = json.loads((checkpoint / 'hfd_fused_delta_manifest.json').read_text())
required = ['model.safetensors', 'optimizer.bin', 'scheduler.bin',
'custom_checkpoint_0.pkl'] + [f'random_states_{r}.pkl' for r in range(world)]
return (marker['source_group_complete'] and marker['global_step'] == int(checkpoint.name[5:])
and marker['train_infer_contract'] == manifest['contract']
and all((checkpoint / n).stat().st_size > 0 for n in required))
except (OSError, ValueError, KeyError):
return False
def synchronize(run, destination, world, keep=2):
"""Mirror public run artifacts and publish only fully verified immutable ckpts.
Checkpoint leases prevent training retention deleting an in-flight source.
Two committed generations are retained, only after the new one is committed.
No deletion occurs outside destination/checkpoints/ckpt_<step>.
"""
run, destination = Path(run), Path(destination)
destination.mkdir(parents=True, exist_ok=True)
owner = destination / '.hfd-backup-owner.json'
identity = dict(source=str(run.resolve()), format='hfd-dlc-mirror-v1')
if owner.exists() and json.loads(owner.read_text()) != identity:
raise ValueError('Backup destination belongs to another run')
atomic_json(owner, identity)
# Mutable logs are best-effort snapshots; failure leaves the last good copy.
excluded = {'checkpoints', 'wandb', '.inference_snapshots', '__pycache__'}
def public_files():
for folder, directories, names in os.walk(run):
directories[:] = [d for d in directories if d not in excluded and not d.startswith('.')]
for name in names:
yield Path(folder)/name
for source in public_files():
rel = source.relative_to(run)
if any(part in excluded or part.startswith('.') for part in rel.parts):
continue
if source.is_file() and not source.is_symlink() and source.suffix not in {'.pt', '.bin', '.safetensors'}:
target = destination / rel
stat = source.stat()
if target.exists() and target.stat().st_size == stat.st_size and target.stat().st_mtime_ns == stat.st_mtime_ns:
continue
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(target.name + '.partial')
shutil.copy2(source, temporary)
temporary.replace(target)
candidates = sorted((p for p in (run/'checkpoints').glob('ckpt_*')
if p.name[5:].isdigit()), key=lambda p: int(p.name[5:]), reverse=True)
step = None
for checkpoint in candidates:
if not ready(checkpoint, world):
continue
final = destination/'checkpoints'/checkpoint.name
if (final/'backup_complete.json').exists():
step = int(checkpoint.name[5:])
break
with inference_checkpoint_lease(checkpoint):
if not ready(checkpoint, world):
continue
staging = final.with_name('.'+final.name+'.uploading')
files = {}
for source in sorted(checkpoint.rglob('*')):
if source.is_file() and not source.is_symlink():
rel = str(source.relative_to(checkpoint))
files[rel] = copy_verified(source, staging/rel)
if not files:
raise OSError('Empty checkpoint')
atomic_json(staging/'backup_complete.json', dict(step=int(checkpoint.name[5:]), files=files))
if final.exists():
raise FileExistsError(f'Uncommitted destination exists: {final}')
staging.rename(final)
step = int(checkpoint.name[5:])
break
if step is not None:
atomic_json(destination/'latest_backup.json', dict(step=step, checkpoint=f'checkpoints/ckpt_{step}', updated=time.time()))
committed = sorted((p for p in (destination/'checkpoints').glob('ckpt_*')
if p.name[5:].isdigit() and (p/'backup_complete.json').is_file()),
key=lambda p: int(p.name[5:]), reverse=True)
for old in committed[max(2, keep):]:
remove_checkpoint_unless_in_use(old)
# Only our owned, unpublished staging generations older than the now
# verified checkpoint can be discarded; never touch another run.
for old in (destination/'checkpoints').glob('.ckpt_*.uploading'):
suffix = old.name[len('.ckpt_'):-len('.uploading')]
if suffix.isdigit() and int(suffix) <= step and old.is_dir() and not old.is_symlink():
shutil.rmtree(old)
return step
def restore(backup, target, world):
"""Restore one committed full state, validating every recorded checksum."""
backup, target = Path(backup), Path(target)
pointer = json.loads((backup/'latest_backup.json').read_text())
name = f"ckpt_{int(pointer['step'])}"
source = backup/'checkpoints'/name
receipt = json.loads((source/'backup_complete.json').read_text())
rng_ranks = {p.name for p in source.glob('random_states_*.pkl')}
if rng_ranks != {f'random_states_{r}.pkl' for r in range(world)} or not ready(source, world):
raise ValueError('Backup is incomplete or has a different rank count')
checkpoint = target/'checkpoints'/name
for rel, expected in receipt['files'].items():
if Path(rel).is_absolute() or '..' in Path(rel).parts:
raise ValueError('Invalid backup member')
actual = copy_verified(source/rel, checkpoint/rel)
if actual != expected:
raise OSError(f'Backup checksum mismatch: {rel}')
for filename in ('run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json'):
copy_verified(backup/filename, target/filename)
shutil.copytree(backup/'control_tokenizer', target/'control_tokenizer', dirs_exist_ok=True)
if not ready(checkpoint, world):
raise OSError('Restored checkpoint is incomplete')
return checkpoint