RealArDor/NanoJev-SimuNRGTrans / source /scripts /evaluate_regression_decisions.py
RealArDor's picture
download
raw
5.17 kB
#!/usr/bin/env python3
"""Read-only V3-on-V2 non-grid regression diagnostic; never train or select a model."""
import argparse
from collections import Counter, defaultdict
import hashlib
import json
from pathlib import Path
import time
from train_pipeline_decisions import (
distribution_metrics, evaluate_pipeline, load_training_examples, read_training_records,
)
from predict_toy_decisions import DecisionPredictor
def digest(path):
with Path(path).open('rb') as handle:
return hashlib.file_digest(handle, 'sha256').hexdigest()
def summarize_predictions(rows):
"""Reuse the training evaluator's exact stable-logit CE/KL/TV definitions."""
buckets = defaultdict(list)
for row in rows:
for axis, value in [('overall', 'all'), ('family', row['family_id']),
('type', row['type']), ('target_kind', row.get('gold_probs_kind') or 'hard_gold_one_hot')]:
buckets[(row['split'], axis, value)].append(row)
result = {}
for (split, axis, value), group in buckets.items():
metrics = {}
for target, field in [('gold_distribution', 'gold_distribution_probs'), ('teacher', 'teacher_probs')]:
values = [distribution_metrics(row.get(field), row['student_logits']) for row in group]
values = [x for x in values if x is not None]
metrics[target] = {'n': len(values), **{key: sum(x[key] for x in values) / len(values) if values else None for key in ['ce', 'kl', 'tv']}}
result.setdefault(split, {}).setdefault(axis, {})[value] = {'questions': len(group), 'targets': metrics}
return result
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument('--input', required=True)
ap.add_argument('--checkpoint-dir', required=True)
ap.add_argument('--output-dir', required=True)
ap.add_argument('--precision', choices=['bf16', 'fp32'], default='bf16')
ap.add_argument('--microbatch-questions', type=int, default=4)
ap.add_argument('--max-microbatch-tokens', type=int, default=6000)
ap.add_argument('--max-length', type=int, default=512)
ap.add_argument('--disable-native-triton', action='store_true')
args = ap.parse_args()
out = Path(args.output_dir)
if (out / 'summary.json').exists():
raise ValueError('Refusing to overwrite a completed regression diagnostic')
out.mkdir(parents=True, exist_ok=True)
records, files = read_training_records(args.input)
selected = [r for r in records if r['split'] in {'test', 'ood'}
and r.get('metadata', {}).get('environment_state', {}).get('game') != 'grid_navigation']
if not selected or any('grid' in r['family_id'] for r in selected):
raise ValueError('Empty or incorrectly filtered non-grid regression cohort')
filtered = out / 'input.jsonl'
filtered.write_text(''.join(json.dumps(r, ensure_ascii=False, allow_nan=False) + '\n' for r in selected))
checkpoint = Path(args.checkpoint_dir)
before_hash = digest(checkpoint / 'best.safetensors')
start = time.perf_counter()
engine = DecisionPredictor(checkpoint, max_length=args.max_length,
disable_native_triton=args.disable_native_triton, precision=args.precision)
examples, _ = load_training_examples(filtered, engine.tokenizer, args.max_length)
metrics = {}
for split in ['test', 'ood']:
group = [ex for ex in examples if ex['split'] == split]
metrics[split] = evaluate_pipeline(engine.model, group, engine.tokenizer.pad_token_id,
args, 'gold_distribution', out / f'predictions_{split}.jsonl')
rows = [json.loads(line) for split in ['test', 'ood']
for line in (out / f'predictions_{split}.jsonl').read_text().splitlines() if line.strip()]
(out / 'predictions.jsonl').write_text(''.join(json.dumps(r, ensure_ascii=False, allow_nan=False) + '\n' for r in rows))
summary = {'name': checkpoint.name, 'diagnostic': 'Historical V2 held-out non-grid regression only; not new generalization evidence or a model-selection set.',
'checkpoint_sha256': before_hash, 'source_sha256': {str(p): digest(p) for p in files},
'filtered_input_sha256': digest(filtered), 'script_sha256': digest(__file__),
'selected_records': len(selected), 'questions': len(examples),
'record_counts_by_split': dict(Counter(r['split'] for r in selected)),
'metrics_by_split': metrics, 'grouped_metrics': summarize_predictions(rows),
'precision': args.precision, 'temperature': 1.0, 'temperature_fitted': False,
'training_steps': 0, 'network_model_calls': 0, 'evaluation_seconds': time.perf_counter() - start,
'max_gpu_allocated_gb': engine._torch.cuda.max_memory_allocated() / 1e9}
(out / 'summary.json').write_text(json.dumps(summary, ensure_ascii=False, indent=2, allow_nan=False) + '\n')
print(json.dumps({k: summary[k] for k in ['name', 'checkpoint_sha256', 'selected_records', 'questions', 'evaluation_seconds', 'max_gpu_allocated_gb']}), flush=True)
if __name__ == '__main__':
main()

Xet Storage Details

Size:
5.17 kB
·
Xet hash:
cb0e7b8f0b312d481352587ad78712bbc468cc8a902eae64922d6c3f6f1964e2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.