RealArDor/NanoJev-SimuNRGTrans / source /scripts /summarize_unified_cohort.py
RealArDor's picture
download
raw
8.2 kB
#!/usr/bin/env python3
"""Compare verified runs on an explicit, fully covered reference cohort.
All source files and manifests are validated before filtering. The reference
file defines the cohort; no intersection or outcome-dependent selection is used.
Source SHA256 values always identify the original complete files, not a subset.
No episodes, manifests, model outputs, or training files are modified.
"""
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import sys
import summarize_unified_games as summary_tools
def cohort_view(run, reference):
selected = set(reference["cases"])
missing = sorted(selected - set(run["cases"]))
if missing:
raise ValueError(f"run {run['name']!r} does not cover the explicit cohort; missing case IDs: {missing}")
changed = [cid for cid in sorted(selected)
if summary_tools.canon(run["cases"][cid]) != summary_tools.canon(reference["cases"][cid])]
if changed:
raise ValueError(f"run {run['name']!r} has different case definitions for cohort IDs: {changed}")
source_selection = []
for source in run["sources"]:
included = sorted(selected & set(source["case_ids"]))
source_selection.append({**source, "source_case_count": len(source["case_ids"]),
"included_case_count": len(included),
"excluded_case_count": len(source["case_ids"]) - len(included),
"included_case_ids": included,
"sha256_scope": "original_complete_source_file"})
active_sources = [source for source in source_selection if source["included_case_count"]]
policy_ids = list(dict.fromkeys(source["continuation_policy_id"] for source in active_sources))
common_policy = {key: value for key, value in active_sources[0]["policy"].items()
if all(key in source["policy"] and summary_tools.canon(source["policy"][key]) == summary_tools.canon(value)
for source in active_sources[1:])}
view = {**run, "cases": {cid: run["cases"][cid] for cid in sorted(selected)},
"case_policies": {cid: run["case_policies"][cid] for cid in sorted(selected)},
"episodes": [episode for episode in run["episodes"] if episode["id"] in selected],
"sources": active_sources, "policy": common_policy,
"continuation_policy_ids": policy_ids,
"continuation_policy_id": policy_ids[0] if len(policy_ids) == 1 else None}
selection = {"source_case_count": len(run["cases"]), "included_case_count": len(selected),
"excluded_case_count": len(run["cases"]) - len(selected), "sources": source_selection}
return view, selection
def build_summary(run_specs, cohort_from, training_specs=None):
if not run_specs:
raise ValueError("At least one --run NAME=path is required")
reference = summary_tools.load_run_sources("cohort_reference", cohort_from)
complete_runs = [summary_tools.load_run_sources(name, paths) for name, paths in run_specs.items()]
views, selections = [], {}
for run in complete_runs:
view, selection = cohort_view(run, reference)
views.append(view)
selections[run["name"]] = selection
by_name = {run["name"]: run for run in views}
cohort = summary_tools.check_cohort(views)
cohort.update({"selection_rule": "exact_case_ids_from_verified_reference; complete_coverage_required",
"reference_episodes_path": str(cohort_from),
"reference_episodes_sha256": reference["episodes_sha256"],
"reference_manifest_sha256": reference["manifest_sha256"],
"selected_case_ids": sorted(reference["cases"]),
"selection_uses_outcomes": False})
summaries = {run["name"]: summary_tools.summarize_run(run) for run in views}
training = {}
for name, path in (training_specs or {}).items():
if name not in by_name:
raise ValueError(f"--training name {name!r} does not refer to a --run name")
training[name] = summary_tools.load_training(name, path, by_name[name])
return {
"tool": "scripts/summarize_unified_cohort.py",
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"inputs": {run["name"]: {key: run[key] for key in
("episodes_path", "episodes_sha256", "manifest_path", "manifest_sha256")}
| {"sources": selections[run["name"]]["sources"],
"sha256_scope": "original_complete_source_files"} for run in complete_runs},
"policies": {run["name"]: {key: run[key] for key in
("continuation_policy_id", "continuation_policy_ids", "policy", "sources")} for run in views},
"cohort": cohort, "source_selection": selections,
"policy_differences": summary_tools.policy_differences(views), "runs": summaries,
"comparison": summary_tools.comparison_tables(list(by_name), summaries),
"training_diagnostics": training,
"notes": [
"The explicit reference cohort is selected by case ID, never by success or by the intersection of available results.",
"Every original source is hash-verified before filtering. Every compared run must cover the entire reference cohort with identical case definitions.",
"Input episodes_sha256 and manifest_sha256 identify the original complete source files. No subset episodes file or rewritten manifest is produced.",
"The cohort SHA256 identifies canonical selected case definitions; it is not a trajectory or source-file SHA256.",
"Policies with no selected cases remain in input provenance but are excluded from selected-cohort policy comparisons.",
*summary_tools.NOTES,
],
}
def render_markdown(summary):
text = summary_tools.render_markdown(summary).replace(
"Generated by scripts/summarize_unified_games.py", "Generated by scripts/summarize_unified_cohort.py", 1)
rows = ["", "## Explicit cohort selection", "",
"The source hashes above refer to the complete original files. Only the explicit reference case IDs enter these comparisons; missing cases are errors.", "",
f"Reference: `{summary['cohort']['reference_episodes_path']}`", "",
"| Run | Source cases | Included cases | Excluded cases |",
"|---|---:|---:|---:|"]
for name, selection in summary["source_selection"].items():
rows.append(f"| {name} | {selection['source_case_count']} | {selection['included_case_count']} | {selection['excluded_case_count']} |")
return text + "\n".join(rows) + "\n"
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--cohort-from", required=True, help="completed reference episodes JSONL with its original manifest")
parser.add_argument("--run", action="append", required=True, metavar="NAME=EPISODES_JSONL",
help="repeat a name to combine disjoint, independently verified sources")
parser.add_argument("--training", action="append", default=[], metavar="NAME=TRAINER_SUMMARY_JSON")
parser.add_argument("--output-dir", required=True, type=Path)
args = parser.parse_args(argv)
try:
if args.output_dir.exists():
raise ValueError("--output-dir must be a fresh directory")
summary = build_summary(summary_tools.parse_named(args.run, "--run"), args.cohort_from,
summary_tools.parse_named(args.training, "--training"))
args.output_dir.mkdir(parents=True, exist_ok=False)
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2, allow_nan=False) + "\n")
(args.output_dir / "summary.md").write_text(render_markdown(summary))
except (ValueError, OSError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
print(f"Wrote {args.output_dir / 'summary.json'} and {args.output_dir / 'summary.md'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

Xet Storage Details

Size:
8.2 kB
·
Xet hash:
5f91ca6264ff5f17d003859014212ceeb4e2920c10f0a9846c03a0a5c2e1bf4e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.