Buckets:
| """Atomic per-query validation bundles, bound to an exact actor and sample set.""" | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import uuid | |
| import torch | |
| from shared.artifacts import file_sha256, write_json | |
| def validation_directory(run_dir, stem, identity): | |
| digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() | |
| return Path(run_dir) / "validation" / f"{stem}-{digest[:16]}" | |
| def read_sample(root, position, sample_id, identity): | |
| path = Path(root) / "samples" / f"{position:06d}" | |
| if not path.exists(): | |
| return None | |
| record = json.loads((path / "result.json").read_text()) | |
| if (record["identity"] != identity or record["sample_id"] != sample_id | |
| or record["position"] != position | |
| or record["trajectory_sha256"] != file_sha256(path / "trajectories.pt")): | |
| raise ValueError(f"validation sample integrity/identity mismatch: {path}") | |
| return record["metrics_state"] | |
| def save_sample(root, position, sample_id, identity, scored, metrics_state, tokenizer): | |
| samples = Path(root) / "samples" | |
| samples.mkdir(parents=True, exist_ok=True) | |
| final = samples / f"{position:06d}" | |
| temporary = samples / f".tmp-{position:06d}-{uuid.uuid4().hex}" | |
| temporary.mkdir() | |
| torch.save(tuple(value.trajectory for value in scored), temporary / "trajectories.pt") | |
| write_json(temporary / "result.json", { | |
| "identity": identity, "sample_id": sample_id, "position": position, | |
| "trajectory_sha256": file_sha256(temporary / "trajectories.pt"), | |
| "metrics_state": metrics_state, | |
| "rollouts": [{ | |
| "group_index": value.trajectory.group_index, | |
| "stop_reason": value.trajectory.stop_reason, | |
| "protocol_valid": value.protocol.valid, | |
| "protocol_failure": value.protocol.failure_reason, | |
| "correct": value.verifier_correct, | |
| "reward": value.reward.total, | |
| "ig_reward": value.reward.ig, | |
| "text": tokenizer.decode(value.trajectory.input_ids.tolist(), skip_special_tokens=False), | |
| } for value in scored], | |
| }) | |
| # The complete directory is the commit: interrupted temporary bundles are ignored. | |
| temporary.rename(final) | |
Xet Storage Details
- Size:
- 2.23 kB
- Xet hash:
- fe3be373af488750e4667d568f06540706e40e8d02ec2b97192a661d9de157c7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.