Buckets:
| """Audit the real GRPO checkpoint-load boundary on allocated GPUs, then stop.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| from unittest.mock import patch | |
| from rl.config import load_config | |
| from rl.distributed import launch_configured_workers | |
| from rl import trainer | |
| from shared.artifacts import write_json | |
| class StartupComplete(Exception): | |
| pass | |
| def worker(config): | |
| import os | |
| import torch | |
| rank = int(os.environ.get("RANK", "0")) | |
| original = trainer.load_rl_checkpoint | |
| run = Path(config.run.output_dir) | |
| def inspect(checkpoint, **kwargs): | |
| payload = torch.load(Path(checkpoint) / "training_state.pt", map_location="cpu", weights_only=False, mmap=True) | |
| before, after = payload["provenance_fingerprints"], kwargs["provenance_fingerprints"] | |
| difference = {key: {"expected": before.get(key), "actual": after.get(key)} | |
| for key in sorted(before.keys() | after.keys()) if before.get(key) != after.get(key)} | |
| write_json(run / f"provenance-rank-{rank}.json", {"expected": before, "actual": after, "difference": difference}) | |
| print("RL_STARTUP_PROVENANCE " + json.dumps({"rank": rank, "difference": difference}), flush=True) | |
| return original(checkpoint, **kwargs) | |
| with patch.object(trainer, "load_rl_checkpoint", side_effect=inspect), patch.object( | |
| trainer, "collect_logical_batch", side_effect=StartupComplete | |
| ): | |
| try: | |
| trainer.train(config) | |
| except StartupComplete: | |
| write_json(run / f"startup-passed-rank-{rank}.json", {"passed": True}) | |
| print(f"RL_STARTUP_PASSED rank={rank}", flush=True) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", required=True) | |
| parser.add_argument("--output-dir", required=True) | |
| args = parser.parse_args() | |
| config = load_config(args.config, [f"run.output_dir={args.output_dir}"]) | |
| launch_configured_workers(config, worker) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 2.02 kB
- Xet hash:
- 7129943484252e5031585e9ad6a6656cecf07649ab75dc2ee496e6122f534434
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.