Buckets:
| # 中文源码导读(2026-09-15)|阅读顺序 056|本地工程 | |
| # 来源:官方同路径无此文件。 | |
| # 作用:核对日志、树和完整 checkpoint,检查真实策略版本与 Adam 更新,不能只看循环步数。 | |
| # 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。 | |
| """Check durable state, sampling weights, tree and actual optimizer evidence.""" | |
| import argparse | |
| import json | |
| import math | |
| from pathlib import Path | |
| from ttt_discover.runtime.qwen_profile import MODEL as QWEN_MODEL, adapter_layout | |
| def audit_dp_runtime_states(directory, state, world_size): | |
| """Dense DP saves one shared adapter/Adam plus each rank's private RNG. | |
| Runtime records prove complete resumable rank state, not equality of unsaved | |
| replica tensors. Gradient synchronization is tested separately. | |
| """ | |
| import torch | |
| if state.get("rank") != 0 or state.get("world_size") != world_size: | |
| raise RuntimeError("Qwen shared checkpoint rank/world size mismatch") | |
| metadata = state.get("metadata", {}) | |
| if metadata.get("parallelism") != "data_parallel" or metadata.get("model_name") != QWEN_MODEL: | |
| raise RuntimeError("Qwen checkpoint must contain DP metadata") | |
| if not state.get("rng_state"): | |
| raise RuntimeError("Qwen shared checkpoint is missing RNG state") | |
| for rank in range(1, world_size): | |
| replica = torch.load(Path(directory) / f"runtime-rank-{rank:02d}.pt", map_location="cpu", weights_only=False) | |
| if replica.get("rank") != rank or replica.get("world_size") != world_size: | |
| raise RuntimeError(f"Qwen DP runtime rank/world size mismatch on rank {rank}") | |
| if replica.get("policy_version") != state["policy_version"]: | |
| raise RuntimeError(f"Qwen DP policy version differs on rank {rank}") | |
| if replica.get("metadata") != state["metadata"] or not replica.get("rng_state"): | |
| raise RuntimeError(f"Qwen DP runtime metadata/RNG missing or inconsistent on rank {rank}") | |
| def verify_run(run_dir): | |
| run_dir = Path(run_dir).resolve() | |
| c = json.loads((run_dir / "resolved_config.json").read_text()) | |
| log_dir = run_dir / "tinker_log" / c["experiment_name"] | |
| train_log = "\n".join(path.read_text(errors="replace") for path in | |
| (log_dir / "train.log", run_dir / "launcher.log") if path.exists()) | |
| if "Training completed successfully" not in train_log: | |
| raise RuntimeError("No successful training-completion log") | |
| checkpoints = run_dir / "checkpoints" | |
| target = c["logged_batches"] | |
| tree_path = log_dir / f"puct_sampler_step_{target:06d}.json" | |
| if not tree_path.is_file(): | |
| raise RuntimeError(f"Missing matching tree snapshot: {tree_path}") | |
| tree = json.loads(tree_path.read_text()) | |
| if int(tree.get("step", -1)) != target: | |
| raise RuntimeError("Tree step does not match checkpoint boundary") | |
| records = [json.loads(line) for line in (log_dir / "checkpoints.jsonl").read_text().splitlines() if line.strip()] | |
| if not any(row.get("name") == "final" and row.get("batch") == target and "state_path" in row for row in records): | |
| raise RuntimeError("No complete final checkpoint index at the target boundary") | |
| # Loop saves numeric boundaries before the next batch and 'final' at exit. | |
| final = checkpoints / "final" | |
| world_size = len(c["train_gpus"]) | |
| qwen = c["model_name"] == QWEN_MODEL | |
| rank_files = [f'{"runtime" if qwen and i > 0 else "state"}-rank-{i:02d}.pt' for i in range(world_size)] | |
| for name in ["checkpoint_complete"] + rank_files: | |
| p = final / name | |
| if not p.is_file() or (name.endswith(".pt") and p.stat().st_size < 1024): | |
| raise RuntimeError(f"Missing/incomplete training shard: {p}") | |
| sampler = checkpoints / "sampler_weights/final" | |
| for name in ("checkpoint_complete", "adapter_model.safetensors", "adapter_config.json"): | |
| if not (sampler / name).is_file(): | |
| raise RuntimeError(f"Missing sampling checkpoint component: {sampler / name}") | |
| import torch | |
| # This is a CPU audit, separate from the trainer processes. Avoid spawning | |
| # hundreds of CPU threads for each relatively small LoRA tensor. | |
| torch.set_num_threads(min(8, torch.get_num_threads())) | |
| from safetensors import safe_open | |
| with safe_open(sampler / "adapter_model.safetensors", framework="pt", device="cpu") as handle: | |
| tensor_count = len(handle.keys()) | |
| parameter_count = sum(math.prod(handle.get_slice(key).get_shape()) for key in handle.keys()) | |
| for key in handle.keys(): | |
| if not bool(torch.isfinite(handle.get_tensor(key)).all()): | |
| raise RuntimeError(f"Non-finite sampling adapter tensor: {key}") | |
| expected_tensors, expected_parameters = adapter_layout(c["model_name"]) | |
| if (tensor_count, parameter_count) != (expected_tensors, expected_parameters): | |
| raise RuntimeError(f'Incomplete {c["model_name"]} adapter: {tensor_count} tensors, {parameter_count} parameters') | |
| # Inspect one shard's counter/Adam state on CPU; existence alone isn't an update. | |
| state = torch.load(final / "state-rank-00.pt", map_location="cpu", weights_only=False) | |
| no_rl = c.get("method") == "no-rl-reflection" | |
| modular_root = c.get("method") == "modular-root" | |
| if no_rl: | |
| if int(state["policy_version"]) != 0 or state["optimizer"]["state"]: | |
| raise RuntimeError("No-RL run unexpectedly updated policy or optimizer") | |
| elif int(state["policy_version"]) < c["updates"] or not state["optimizer"]["state"]: | |
| raise RuntimeError("Checkpoint has no completed optimizer update") | |
| adam_steps = [float(s["step"]) for s in state["optimizer"]["state"].values() if "step" in s] | |
| if not no_rl and (not adam_steps or min(adam_steps) < c["updates"]): | |
| raise RuntimeError("Adam step counter is behind the requested updates") | |
| learned_b = any("lora_B" in key and bool(tensor.count_nonzero()) for key, tensor in state["adapter"].items()) | |
| if no_rl and learned_b: | |
| raise RuntimeError("No-RL run changed zero-initialized LoRA B tensors") | |
| if not no_rl and not learned_b: | |
| raise RuntimeError("All zero-initialized LoRA B tensors remain zero; no learned update observed") | |
| if qwen: | |
| audit_dp_runtime_states(final, state, world_size) | |
| result = dict(task=c["task"], smoke=c["smoke"], logged_batches=target, | |
| policy_version=int(state["policy_version"]), | |
| min_adam_step=min(adam_steps) if adam_steps else None, | |
| rank_shards=world_size, checkpoint=str(final), sampler_checkpoint=str(sampler), | |
| sampler_tensors=tensor_count, sampler_parameters=parameter_count, nonzero_lora_B=learned_b, | |
| finite_sampler_weights=True, | |
| tree_snapshot=str(tree_path)) | |
| if qwen: | |
| result.update(dp_runtime_records_complete=True, shared_adapter_optimizer_shards=1, | |
| private_runtime_shards=world_size - 1) | |
| if no_rl: | |
| initial = checkpoints / "000001" | |
| reflection = log_dir / "circle_no_rl_reflection/reflection_step_000001.json" | |
| for path in (initial / "checkpoint_complete", reflection): | |
| if not path.is_file(): | |
| raise RuntimeError(f"Missing admitted Step-0 boundary evidence: {path}") | |
| result.update( | |
| no_rl=True, | |
| optimizer_updates=0, | |
| imported_step0=True, | |
| imported_step0_checkpoint=str(initial), | |
| imported_step0_reflection=str(reflection), | |
| ) | |
| elif ( | |
| c["root"] == "fresh_baseline" | |
| and c.get("method", "single-anchor") != "baseline" | |
| ): | |
| initial = checkpoints / "000001" | |
| if not (initial / "checkpoint_complete").is_file(): | |
| raise RuntimeError("Missing durable checkpoint after root selection") | |
| initial_state = torch.load(initial / "state-rank-00.pt", map_location="cpu", weights_only=False) | |
| initial_b_changed = any("lora_B" in key and bool(value.count_nonzero()) | |
| for key, value in initial_state["adapter"].items()) | |
| if int(initial_state["policy_version"]) != 0 or initial_state["optimizer"]["state"] or initial_b_changed: | |
| raise RuntimeError("Root-selection batch unexpectedly changed the model or optimizer") | |
| if modular_root: | |
| imported_tree = log_dir / "puct_sampler_step_000001.json" | |
| if not imported_tree.is_file(): | |
| raise RuntimeError( | |
| "Missing sampler snapshot for the admitted modular root" | |
| ) | |
| imported_payload = json.loads(imported_tree.read_text()) | |
| imported_states = imported_payload.get("initial_states", []) | |
| if len(imported_states) != c["groups_per_batch"]: | |
| raise RuntimeError("Admitted modular root parent count is wrong") | |
| if any( | |
| state.get("phase") != "search" | |
| or state.get("bootstrap_winner_id") | |
| != "f504f041-7649-496b-a70a-02e8d17e74ee" | |
| or not math.isclose( | |
| float(state.get("value")), | |
| 2.6310935781919955, | |
| rel_tol=0.0, | |
| abs_tol=1e-10, | |
| ) | |
| for state in imported_states | |
| ): | |
| raise RuntimeError("Admitted modular root identity/score changed") | |
| if "Starting fresh policy at imported sampler boundary 1" not in train_log: | |
| raise RuntimeError("Missing imported modular-root boundary log") | |
| result.update( | |
| modular_root=True, | |
| imported_step0=True, | |
| imported_step0_tree=str(imported_tree), | |
| imported_step0_checkpoint=str(initial), | |
| imported_parent_count=len(imported_states), | |
| ) | |
| else: | |
| if "Bootstrap root selected; skipping policy update" not in train_log: | |
| raise RuntimeError("Missing explicit root-selection update-skip log") | |
| result["bootstrap_untrained"] = True | |
| result["bootstrap_checkpoint"] = str(initial) | |
| return result | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("run_dir", type=Path) | |
| print(json.dumps(verify_run(parser.parse_args().run_dir), indent=2)) | |
Xet Storage Details
- Size:
- 10.3 kB
- Xet hash:
- ee3e999f0e63a3a7d753f4308c6973f67147a22d4d9e77267133b9b443f38fac
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.