zefancainlp's picture
download
raw
10.3 kB
# 中文源码导读(2026-09-15)|阅读顺序 056|本地工程
# 来源:官方同路径无此文件。
# 作用:核对日志、树和完整 checkpoint,检查真实策略版本与 Adam 更新,不能只看循环步数。
# 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。
"""Check durable state, sampling weights, tree and actual optimizer evidence."""
import argparse
import json
import math
from pathlib import Path
from ttt_discover.runtime.qwen_profile import MODEL as QWEN_MODEL, adapter_layout
def audit_dp_runtime_states(directory, state, world_size):
"""Dense DP saves one shared adapter/Adam plus each rank's private RNG.
Runtime records prove complete resumable rank state, not equality of unsaved
replica tensors. Gradient synchronization is tested separately.
"""
import torch
if state.get("rank") != 0 or state.get("world_size") != world_size:
raise RuntimeError("Qwen shared checkpoint rank/world size mismatch")
metadata = state.get("metadata", {})
if metadata.get("parallelism") != "data_parallel" or metadata.get("model_name") != QWEN_MODEL:
raise RuntimeError("Qwen checkpoint must contain DP metadata")
if not state.get("rng_state"):
raise RuntimeError("Qwen shared checkpoint is missing RNG state")
for rank in range(1, world_size):
replica = torch.load(Path(directory) / f"runtime-rank-{rank:02d}.pt", map_location="cpu", weights_only=False)
if replica.get("rank") != rank or replica.get("world_size") != world_size:
raise RuntimeError(f"Qwen DP runtime rank/world size mismatch on rank {rank}")
if replica.get("policy_version") != state["policy_version"]:
raise RuntimeError(f"Qwen DP policy version differs on rank {rank}")
if replica.get("metadata") != state["metadata"] or not replica.get("rng_state"):
raise RuntimeError(f"Qwen DP runtime metadata/RNG missing or inconsistent on rank {rank}")
def verify_run(run_dir):
run_dir = Path(run_dir).resolve()
c = json.loads((run_dir / "resolved_config.json").read_text())
log_dir = run_dir / "tinker_log" / c["experiment_name"]
train_log = "\n".join(path.read_text(errors="replace") for path in
(log_dir / "train.log", run_dir / "launcher.log") if path.exists())
if "Training completed successfully" not in train_log:
raise RuntimeError("No successful training-completion log")
checkpoints = run_dir / "checkpoints"
target = c["logged_batches"]
tree_path = log_dir / f"puct_sampler_step_{target:06d}.json"
if not tree_path.is_file():
raise RuntimeError(f"Missing matching tree snapshot: {tree_path}")
tree = json.loads(tree_path.read_text())
if int(tree.get("step", -1)) != target:
raise RuntimeError("Tree step does not match checkpoint boundary")
records = [json.loads(line) for line in (log_dir / "checkpoints.jsonl").read_text().splitlines() if line.strip()]
if not any(row.get("name") == "final" and row.get("batch") == target and "state_path" in row for row in records):
raise RuntimeError("No complete final checkpoint index at the target boundary")
# Loop saves numeric boundaries before the next batch and 'final' at exit.
final = checkpoints / "final"
world_size = len(c["train_gpus"])
qwen = c["model_name"] == QWEN_MODEL
rank_files = [f'{"runtime" if qwen and i > 0 else "state"}-rank-{i:02d}.pt' for i in range(world_size)]
for name in ["checkpoint_complete"] + rank_files:
p = final / name
if not p.is_file() or (name.endswith(".pt") and p.stat().st_size < 1024):
raise RuntimeError(f"Missing/incomplete training shard: {p}")
sampler = checkpoints / "sampler_weights/final"
for name in ("checkpoint_complete", "adapter_model.safetensors", "adapter_config.json"):
if not (sampler / name).is_file():
raise RuntimeError(f"Missing sampling checkpoint component: {sampler / name}")
import torch
# This is a CPU audit, separate from the trainer processes. Avoid spawning
# hundreds of CPU threads for each relatively small LoRA tensor.
torch.set_num_threads(min(8, torch.get_num_threads()))
from safetensors import safe_open
with safe_open(sampler / "adapter_model.safetensors", framework="pt", device="cpu") as handle:
tensor_count = len(handle.keys())
parameter_count = sum(math.prod(handle.get_slice(key).get_shape()) for key in handle.keys())
for key in handle.keys():
if not bool(torch.isfinite(handle.get_tensor(key)).all()):
raise RuntimeError(f"Non-finite sampling adapter tensor: {key}")
expected_tensors, expected_parameters = adapter_layout(c["model_name"])
if (tensor_count, parameter_count) != (expected_tensors, expected_parameters):
raise RuntimeError(f'Incomplete {c["model_name"]} adapter: {tensor_count} tensors, {parameter_count} parameters')
# Inspect one shard's counter/Adam state on CPU; existence alone isn't an update.
state = torch.load(final / "state-rank-00.pt", map_location="cpu", weights_only=False)
no_rl = c.get("method") == "no-rl-reflection"
modular_root = c.get("method") == "modular-root"
if no_rl:
if int(state["policy_version"]) != 0 or state["optimizer"]["state"]:
raise RuntimeError("No-RL run unexpectedly updated policy or optimizer")
elif int(state["policy_version"]) < c["updates"] or not state["optimizer"]["state"]:
raise RuntimeError("Checkpoint has no completed optimizer update")
adam_steps = [float(s["step"]) for s in state["optimizer"]["state"].values() if "step" in s]
if not no_rl and (not adam_steps or min(adam_steps) < c["updates"]):
raise RuntimeError("Adam step counter is behind the requested updates")
learned_b = any("lora_B" in key and bool(tensor.count_nonzero()) for key, tensor in state["adapter"].items())
if no_rl and learned_b:
raise RuntimeError("No-RL run changed zero-initialized LoRA B tensors")
if not no_rl and not learned_b:
raise RuntimeError("All zero-initialized LoRA B tensors remain zero; no learned update observed")
if qwen:
audit_dp_runtime_states(final, state, world_size)
result = dict(task=c["task"], smoke=c["smoke"], logged_batches=target,
policy_version=int(state["policy_version"]),
min_adam_step=min(adam_steps) if adam_steps else None,
rank_shards=world_size, checkpoint=str(final), sampler_checkpoint=str(sampler),
sampler_tensors=tensor_count, sampler_parameters=parameter_count, nonzero_lora_B=learned_b,
finite_sampler_weights=True,
tree_snapshot=str(tree_path))
if qwen:
result.update(dp_runtime_records_complete=True, shared_adapter_optimizer_shards=1,
private_runtime_shards=world_size - 1)
if no_rl:
initial = checkpoints / "000001"
reflection = log_dir / "circle_no_rl_reflection/reflection_step_000001.json"
for path in (initial / "checkpoint_complete", reflection):
if not path.is_file():
raise RuntimeError(f"Missing admitted Step-0 boundary evidence: {path}")
result.update(
no_rl=True,
optimizer_updates=0,
imported_step0=True,
imported_step0_checkpoint=str(initial),
imported_step0_reflection=str(reflection),
)
elif (
c["root"] == "fresh_baseline"
and c.get("method", "single-anchor") != "baseline"
):
initial = checkpoints / "000001"
if not (initial / "checkpoint_complete").is_file():
raise RuntimeError("Missing durable checkpoint after root selection")
initial_state = torch.load(initial / "state-rank-00.pt", map_location="cpu", weights_only=False)
initial_b_changed = any("lora_B" in key and bool(value.count_nonzero())
for key, value in initial_state["adapter"].items())
if int(initial_state["policy_version"]) != 0 or initial_state["optimizer"]["state"] or initial_b_changed:
raise RuntimeError("Root-selection batch unexpectedly changed the model or optimizer")
if modular_root:
imported_tree = log_dir / "puct_sampler_step_000001.json"
if not imported_tree.is_file():
raise RuntimeError(
"Missing sampler snapshot for the admitted modular root"
)
imported_payload = json.loads(imported_tree.read_text())
imported_states = imported_payload.get("initial_states", [])
if len(imported_states) != c["groups_per_batch"]:
raise RuntimeError("Admitted modular root parent count is wrong")
if any(
state.get("phase") != "search"
or state.get("bootstrap_winner_id")
!= "f504f041-7649-496b-a70a-02e8d17e74ee"
or not math.isclose(
float(state.get("value")),
2.6310935781919955,
rel_tol=0.0,
abs_tol=1e-10,
)
for state in imported_states
):
raise RuntimeError("Admitted modular root identity/score changed")
if "Starting fresh policy at imported sampler boundary 1" not in train_log:
raise RuntimeError("Missing imported modular-root boundary log")
result.update(
modular_root=True,
imported_step0=True,
imported_step0_tree=str(imported_tree),
imported_step0_checkpoint=str(initial),
imported_parent_count=len(imported_states),
)
else:
if "Bootstrap root selected; skipping policy update" not in train_log:
raise RuntimeError("Missing explicit root-selection update-skip log")
result["bootstrap_untrained"] = True
result["bootstrap_checkpoint"] = str(initial)
return result
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("run_dir", type=Path)
print(json.dumps(verify_run(parser.parse_args().run_dir), indent=2))

Xet Storage Details

Size:
10.3 kB
·
Xet hash:
ee3e999f0e63a3a7d753f4308c6973f67147a22d4d9e77267133b9b443f38fac

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.