RealArDor/NanoJev-SimuNRGTrans / source /scripts /evaluate_unified_checkpoint.py
RealArDor's picture
download
raw
7.96 kB
#!/usr/bin/env python3
"""Evaluate a local unified DecisionModel without optimization or calibration.
--stage selects the metric population weights, not the checkpoint's training
history. Use --stage critic to compare an existing SFT checkpoint's Boolean
predictions with critic runs on the same frozen outcome dataset. Every requested
split retains both policy and outcome rows, as in the trainer's final evaluation.
"""
import argparse
import importlib.metadata
import json
from pathlib import Path
import time
from predict_toy_decisions import local_checkpoint_files
from train_pipeline_decisions import SPLITS, dump, pack_complete_questions
from train_unified_games import (
evaluate_unified, file_sha256, population_weights, prepare_unified_examples,
read_unified_dataset,
)
def parse_args(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", required=True, help="Frozen five-split dataset directory")
parser.add_argument("--checkpoint", required=True, help="Complete local checkpoint bundle")
parser.add_argument("--output-dir", required=True, help="New directory; never overwrite an earlier evaluation")
parser.add_argument("--stage", choices=("sft", "critic"), required=True,
help="Population weighting scheme; does not train or select a checkpoint")
parser.add_argument("--splits", default="dev,calibration,test,ood", help="Comma-separated split names")
parser.add_argument("--balance", choices=("task", "task_role"), default="task")
parser.add_argument("--retention-fraction", type=float, default=.25)
parser.add_argument("--microbatch-questions", type=int, default=8)
parser.add_argument("--max-microbatch-tokens", type=int, default=32768)
parser.add_argument("--max-length", type=int, default=8192)
parser.add_argument("--precision", choices=("bf16", "fp32"), default="bf16")
parser.add_argument("--disable-native-triton", action="store_true")
args = parser.parse_args(argv)
args.splits = [part.strip() for part in args.splits.split(",")]
if not args.splits or any(split not in SPLITS for split in args.splits):
parser.error("--splits must contain names from train,dev,calibration,test,ood")
if len(args.splits) != len(set(args.splits)):
parser.error("--splits must not contain duplicates")
if min(args.microbatch_questions, args.max_length) <= 0 or args.max_microbatch_tokens < 0:
parser.error("Question/max-length limits must be positive; token budget must be nonnegative")
try:
population_weights(args.stage, args.balance, args.retention_fraction)
except ValueError as error:
parser.error(str(error))
return args
def main(argv=None):
args = parse_args(argv)
out = Path(args.output_dir)
if out.exists():
raise ValueError("--output-dir must be a new directory, including when an existing directory is empty")
# These checks are stdlib-only and run before importing Torch or allocating GPU memory.
root, _ = local_checkpoint_files(args.checkpoint)
records, manifest, files, schema_audit = read_unified_dataset(args.input)
data_hashes = {path.name: file_sha256(path) for path in files}
for split, expected in manifest.get("split_sha256", {}).items():
if split not in SPLITS or data_hashes.get(f"{split}.jsonl") != expected:
raise ValueError(f"Dataset file hash disagrees with manifest: {split}")
selected = [row for row in records if row["split"] in args.splits]
if any(not any(row["split"] == split for row in selected) for split in args.splits):
raise ValueError("Every requested split must contain records")
weights = population_weights(args.stage, args.balance, args.retention_fraction)
bundle_files = [root / "config.json", root / "best.safetensors"]
for directory in ("tokenizer", "backbone_config"):
bundle_files.extend(path for path in sorted((root / directory).rglob("*")) if path.is_file())
bundle_hashes = {str(path.relative_to(root)): file_sha256(path) for path in bundle_files}
import torch
from predict_toy_decisions import DecisionPredictor
runtime = DecisionPredictor(root, max_length=args.max_length, precision=args.precision,
disable_native_triton=args.disable_native_triton)
runtime.model.requires_grad_(False)
examples, target_audit = prepare_unified_examples(selected, runtime.tokenizer, args.max_length)
split_examples = {split: [ex for ex in examples if ex["split"] == split] for split in args.splits}
# Preflight complete question groups before producing any prediction file.
for values in split_examples.values():
pack_complete_questions(values, args.microbatch_questions, args.max_microbatch_tokens)
out.mkdir(parents=True, exist_ok=False)
source_files = ("evaluate_unified_checkpoint.py", "train_unified_games.py",
"train_pipeline_decisions.py", "predict_toy_decisions.py", "train_toy_decisions.py")
config = {**vars(args), "schema_version": "nanojev-unified-checkpoint-evaluation-v1",
"stage": "evaluation", "evaluation_weighting_stage": args.stage,
"completed_steps": 0, "temperature": 1.0, "temperature_fitted": False,
"model": runtime.run_config.get("model"), "set_head": runtime.run_config["set_head"],
"data_sha256": data_hashes, "checkpoint_file_sha256": bundle_hashes,
"weights_sha256": bundle_hashes["best.safetensors"],
"implementation_sha256": {name: file_sha256(Path(__file__).with_name(name)) for name in source_files},
"continuation_policy_id": schema_audit["continuation_policy_id"],
"population_weights": {"/".join(cell): weight for cell, weight in weights.items()},
"schema_counts": schema_audit, "parameter_storage": "float32",
"forward_autocast": args.precision,
"deps": {name: importlib.metadata.version(name) for name in ("torch", "transformers", "safetensors")},
"gpu": torch.cuda.get_device_name(0),
"selection": "none; evaluate the supplied checkpoint without parameter updates",
"frozen_continuation_policy_is_updated_by_this_script": False}
dump(out / "config.json", config)
dump(out / "target_audit.json", target_audit)
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
started = time.perf_counter()
metrics = {}
for split in args.splits:
metrics[split] = evaluate_unified(runtime.model, split_examples[split],
runtime.tokenizer.pad_token_id, args, weights,
out / f"predictions_{split}.jsonl", require_all=split == "dev")
print(json.dumps({"split": split, "metrics": metrics[split]}, allow_nan=False), flush=True)
torch.cuda.synchronize()
summary = {"stage": "evaluation", "evaluation_weighting_stage": args.stage,
"completed_steps": 0, "selected_on": "none; supplied checkpoint",
"metrics_by_split": metrics, "weights_sha256": bundle_hashes["best.safetensors"],
"continuation_policy_id": schema_audit["continuation_policy_id"],
"temperature": 1.0, "temperature_fitted": False,
"evaluation_seconds": time.perf_counter() - started,
"max_gpu_allocated_gb": torch.cuda.max_memory_allocated() / 1e9,
"config_sha256": file_sha256(out / "config.json"),
"target_audit_sha256": file_sha256(out / "target_audit.json"),
"prediction_sha256": {split: file_sha256(out / f"predictions_{split}.jsonl") for split in args.splits},
"finished": True}
dump(out / "summary.json", summary)
print(json.dumps({"done": str(out), **summary}, allow_nan=False), flush=True)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
7.96 kB
·
Xet hash:
4ab59e48b523ac359d2e07fded156d3f59dc407eab6913f9f34d54041ecd3af1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.