RealArDor/NanoJev-SimuNRGTrans / source /scripts /summarize_appo_supervision.py
RealArDor's picture
download
raw
17.5 kB
#!/usr/bin/env python3
"""Compare complete APPO-supervision rollouts on an explicit fresh case cohort.
Standard library only; no inference, simulator, network, or credentials. Source
files may include other splits of the same registered cases. Every supplied row
is validated before the requested cohort is selected. Source hashes always refer
to original complete JSONL files, never an invented filtered file.
"""
import argparse
import hashlib
import json
import math
import random
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from summarize_unified_games import SPLITS, canon, group_stats, is_num, load_run, sha256_file
def require(condition, message):
if not condition:
raise ValueError(message)
def digest(value):
return hashlib.sha256(canon(value).encode()).hexdigest()
def rows(path):
with Path(path).open(encoding="utf-8") as handle:
for number, line in enumerate(handle, 1):
if line.strip():
value = json.loads(line)
require(isinstance(value, dict), f"{path}:{number}: expected an object")
yield value
def mcnemar_exact(improved, regressed):
"""Two-sided exact binomial test conditional on discordant episode pairs."""
require(type(improved) is int and type(regressed) is int and min(improved, regressed) >= 0,
"Discordant counts must be nonnegative integers")
n = improved + regressed
if not n:
return 1.0
tail = sum(math.comb(n, k) for k in range(min(improved, regressed) + 1))
return min(1.0, (2 * tail) / (2 ** n))
def paired(reference, other):
a, b = {r["id"]: r for r in reference}, {r["id"]: r for r in other}
require(a and a.keys() == b.keys(), "Paired comparison requires identical nonempty episode IDs")
improved = sorted(k for k in a if b[k]["success"] and not a[k]["success"])
regressed = sorted(k for k in a if a[k]["success"] and not b[k]["success"])
both_success = sorted(k for k in a if a[k]["success"] and b[k]["success"])
delta = {}
for key in ("physical_ticks", "ammo_consumed", "native_reward"):
require(all(key in r["metrics"] for r in [*a.values(), *b.values()]),
f"Paired episodes lack required metric {key}")
delta[key] = {"n": len(a), "mean_other_minus_reference": math.fsum(
b[k]["metrics"][key] - a[k]["metrics"][key] for k in a) / len(a)}
return {"n": len(a), "success_rate_delta": (len(improved) - len(regressed)) / len(a),
"improved": len(improved), "regressed": len(regressed),
"both_success": len(both_success),
"both_failure": len(a) - len(improved) - len(regressed) - len(both_success),
"exact_mcnemar_two_sided_p": mcnemar_exact(len(improved), len(regressed)),
"p_adjustment": "none; descriptive predeclared comparisons",
"improved_case_ids": improved, "regressed_case_ids": regressed,
"paired_metrics": delta}
def controller_check(episode, policy):
"""Recompute the recorded behavior and every actual action from its seeded RNG."""
case = episode["case"]
rng = random.Random(int(digest([case["id"], policy["sampling_seed"]])[:16], 16))
mode = "sample" if policy["engine"] == "random" else policy["controller"]
epsilon = policy["epsilon"]
ticks = 0
rewards = []
for i, step in enumerate(episode["steps"]):
where = f"{case['id']} step {i}"
obs, info = step["observation"], step["info"]
keys = sorted(obs["candidates"])
require(keys == ["left", "noop", "right", "shoot"], f"{where}: Basic candidate mismatch")
require(obs["task"] == "shooting" and obs["step"] == i, f"{where}: observation identity mismatch")
scores = step["scores"]
require(set(scores) == set(keys) and all(is_num(v) and v >= 0 for v in scores.values()),
f"{where}: invalid action scores")
require(abs(math.fsum(scores.values()) - 1) <= 1e-6, f"{where}: scores are not a Choice distribution")
require(step.get("forced") is not True, f"{where}: four-action Basic cannot be forced")
if mode == "sample":
if policy["engine"] == "random":
require(all(abs(scores[k] - .25) <= 1e-12 for k in keys),
f"{where}: random baseline must be uniform")
total = sum(scores.values())
base = {k: scores[k] / total for k in keys}
else:
top = max(keys, key=lambda k: scores[k])
base = {k: float(k == top) for k in keys}
expected = {k: (1 - epsilon) * base[k] + epsilon / len(keys) for k in keys}
actual = step["behavior_probs"]
require(set(actual) == set(keys) and all(is_num(actual[k]) and
abs(actual[k] - expected[k]) <= 1e-12 for k in keys), f"{where}: behavior mismatch")
draw, cumulative, selected = rng.random(), 0.0, keys[-1]
for k in keys:
cumulative += expected[k]
if draw < cumulative:
selected = k
break
require(step["action"] == selected, f"{where}: actual action differs from seeded controller")
if "sampling_draw" in step:
require(step["sampling_draw"] == draw, f"{where}: sampling receipt differs")
requested, physical = info["requested_ticks"], info["actual_ticks"]
require(type(requested) is int and type(physical) is int and
1 <= physical <= requested <= case["spec"].get("frame_skip", 4), f"{where}: invalid ticks")
ticks += physical
rewards.append(step["reward"])
require(is_num(rewards[-1]), f"{where}: invalid reward")
require(info["action_id"] == step["action"] and info["terminated"] is step["terminated"] and
info["truncated"] is False, f"{where}: transition info differs")
require(info["episode_metrics"]["physical_ticks"] == ticks and
info["episode_metrics"]["decisions"] == i + 1, f"{where}: cumulative counters differ")
final = episode["final_info"]
require(episode["steps"] and final == episode["steps"][-1]["info"],
f"{case['id']}: final_info differs from actual terminal transition")
metrics = final["episode_metrics"]
require(type(metrics.get("success")) is bool and metrics["success"] is episode["success"],
f"{case['id']}: metric success differs")
require(is_num(metrics.get("kills")) and (metrics["kills"] > 0) is episode["success"],
f"{case['id']}: success must be observed kill delta > 0")
require(all(is_num(metrics.get(k)) for k in ("physical_ticks", "ammo_consumed", "native_reward")),
f"{case['id']}: missing physical metric")
require(metrics["ammo_consumed"] >= 0 and abs(math.fsum(rewards) - metrics["native_reward"]) <= 1e-9,
f"{case['id']}: reward or ammo accounting mismatch")
return len(episode["steps"])
def load_verified(name, path, registry, case_sha, protocol):
run = load_run(name, path)
require(run["cases_sha256"] == case_sha, f"{name}: source was not collected from the registered fresh case file")
p, evaluation = run["policy"], protocol["evaluation"]
require(digest(p) == run["continuation_policy_id"], f"{name}: policy identity hash mismatch")
require(p.get("engine") in {"checkpoint", "random", "sample_factory_appo"},
f"{name}: unexpected engine, including legacy API runs")
if name == "random":
require(p["engine"] == "random", "The random run must execute the uniform random engine")
else:
require(p["engine"] != "random", f"{name}: named learned run is actually random")
if name == "appo_1111":
require(p["engine"] == "sample_factory_appo", "APPO baseline must use its actual pixel actor")
require(p.get("checkpoint_sha256", {}).get("model") == protocol["expert"]["checkpoint_sha256"],
"APPO baseline checkpoint differs from the frozen expert")
elif p["engine"] == "checkpoint":
require(isinstance(p.get("checkpoint_sha256"), dict) and p["checkpoint_sha256"].get("best.safetensors"),
f"{name}: checkpoint identity missing")
if name == "frozen_sft":
require(p["checkpoint_sha256"]["best.safetensors"] == protocol["initialization"]["weights_sha256"],
"Frozen SFT baseline differs from registered initialization")
for key, expected in (("controller", evaluation["primary_controller"]),
("epsilon", evaluation["epsilon"]),
("sampling_seed", evaluation["sampling_seed"]),
("temperature", 1.0), ("tie_break", "lexicographic_first")):
require(p.get(key) == expected, f"{name}: {key} differs from the registered evaluation")
count = 0
for ep in rows(path):
case = ep["case"]
require(case["id"] in registry and case == registry[case["id"]],
f"{name}: unknown, old, or changed case {case['id']}")
count += controller_check(ep, p)
run["validation"] = {"source_episodes": len(run["episodes"]), "controller_transitions_checked": count,
"policy_identity_verified": True, "physical_replay_performed": False}
return run
def build(protocol_path, case_path, run_paths, reference="frozen_sft", splits=None):
protocol = json.loads(Path(protocol_path).read_text())
require(protocol.get("schema_version") == "nanojev-appo-supervision-v1", "Wrong supervision protocol")
case_path = Path(case_path or protocol["case_file"])
case_sha = sha256_file(case_path)
require(case_sha == protocol["case_sha256"], "Registered case hash mismatch")
expected_names = protocol["evaluation"]["models"]
require(set(run_paths) == set(expected_names),
f"Supply every registered run exactly once: {expected_names}; received {sorted(run_paths)}")
require(reference in run_paths, "Reference run is missing")
case_list = list(rows(case_path))
registry = {c["id"]: c for c in case_list}
require(len(registry) == len(case_list) and registry, "Duplicate or empty case registry")
groups = {}
for c in case_list:
require(c["split"] in SPLITS and c["spec"].get("task") == "shooting" and
c["spec"].get("scenario") == "basic", "This protocol registers Basic shooting cases only")
group = (c["spec"]["scenario"], c["seed"])
require(group not in groups, "Repeated environment seed, including across splits")
groups[group] = c["split"]
require(dict(Counter(c["split"] for c in case_list)) == protocol["case_counts"], "Case counts differ from protocol")
splits = list(protocol["evaluation"]["fresh_splits"] if splits is None else splits)
require(splits and len(set(splits)) == len(splits) and all(s in SPLITS for s in splits), "Invalid split selection")
selected = {cid: c for cid, c in registry.items() if c["split"] in splits}
require(selected and all(any(c["split"] == s for c in selected.values()) for s in splits), "Empty requested split")
loaded, summaries = {}, {}
for name in expected_names:
run = load_verified(name, run_paths[name], registry, case_sha, protocol)
require(set(selected) <= set(run["cases"]), f"{name}: missing registered comparison cases")
eps = [dict(ep, scenario=registry[ep["id"]]["spec"]["scenario"])
for ep in run["episodes"] if ep["id"] in selected]
loaded[name] = eps
by_group = {f"{s}/{g}": group_stats([e for e in eps if e["split"] == s and e["scenario"] == g])
for s, g in sorted({(e["split"], e["scenario"]) for e in eps})}
summaries[name] = {"source": {k: run[k] for k in (
"episodes_path", "manifest_path", "episodes_sha256", "manifest_sha256",
"cases_sha256", "continuation_policy_id", "policy")},
"validation": run["validation"], "included_episodes": len(eps),
"excluded_other_registered_splits": len(run["episodes"]) - len(eps),
"by_split_scenario": by_group, "per_episode": eps}
comparisons = {}
for name in expected_names:
if name == reference:
continue
comparisons[name] = {key: paired(
[e for e in loaded[reference] if f"{e['split']}/{e['scenario']}" == key],
[e for e in loaded[name] if f"{e['split']}/{e['scenario']}" == key])
for key in summaries[reference]["by_split_scenario"]}
return {"schema_version": "nanojev-appo-supervision-summary-v1", "complete": True,
"created_at": datetime.now(timezone.utc).isoformat(),
"protocol_path": str(protocol_path), "protocol_sha256": sha256_file(protocol_path),
"case_file": str(case_path), "case_file_sha256": case_sha,
"cohort_sha256": digest([selected[k] for k in sorted(selected)]),
"cohort_case_ids": sorted(selected), "splits": splits, "reference": reference,
"runs": summaries, "paired_vs_reference": comparisons,
"implementation_sha256": {Path(__file__).name: sha256_file(__file__),
"summarize_unified_games.py": sha256_file(Path(__file__).with_name("summarize_unified_games.py"))},
"notes": [
"Only registered new cases are included; no historical Jev cohort is pooled into this report.",
"Source hashes identify original complete files. Explicit split selection is a view, not a replacement source.",
"APPO consumes native-rendered recurrent pixels; NanoJev consumes structured visible text. This compares complete policies with different observations.",
"Success means observed kill before the native deadline, not positive native reward. Physical ticks and action decisions differ.",
"Wilson intervals use episodes as units. Exact McNemar tests use paired discordant cases; p-values are unadjusted across comparisons.",
"Paired deltas are other minus reference. Ticks and ammo include failures and early successes and do not independently measure skill.",
"Hard seed 17, hard seed 29 and soft seed 17 remain distinct runs. This is not an estimate over many independent training seeds.",
"Hashes, case definitions, seeded actions and transition counters are verified here; no simulator replay or model inference is performed.",
"Action probabilities describe action choice, not winning probability. Offline expert-action NLL is not evaluated by this rollout summary."]}
def markdown(report):
lines = ["# APPO action-supervision evaluation", "", "Fresh matched episodes; synchronized case definitions and fixed action controllers.", "",
"| Run | Split / scenario | Success | Wilson 95% | Physical ticks | Ammo used |", "|---|---|---:|---|---:|---:|"]
for name, run in report["runs"].items():
for group, stat in run["by_split_scenario"].items():
lo, hi = stat["wilson95"]
lines.append(f"| {name} | {group} | {stat['successes']}/{stat['n']} ({stat['success_rate']:.1%}) | "
f"{lo:.1%}–{hi:.1%} | {stat['metrics']['physical_ticks']['mean']:.2f} | "
f"{stat['metrics']['ammo_consumed']['mean']:.2f} |")
lines += ["", f"Paired comparisons against **{report['reference']}**:", "",
"| Run | Split / scenario | Success delta | Improved / regressed | Exact McNemar p |",
"|---|---|---:|---:|---:|"]
for name, groups in report["paired_vs_reference"].items():
for group, stat in groups.items():
lines.append(f"| {name} | {group} | {stat['success_rate_delta']:+.1%} | "
f"{stat['improved']} / {stat['regressed']} | {stat['exact_mcnemar_two_sided_p']:.5g} |")
lines += ["", *["- " + note for note in report["notes"]], ""]
return "\n".join(lines)
def main(argv=None):
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--protocol", type=Path, default=Path("configs/appo_basic_supervision_v1.json"))
p.add_argument("--cases", type=Path, help="Optional relocated copy; its SHA must match the protocol")
p.add_argument("--run", action="append", required=True, metavar="NAME=EPISODES.jsonl")
p.add_argument("--reference", default="frozen_sft")
p.add_argument("--splits", help="Comma-separated registered splits; default is protocol fresh_splits")
p.add_argument("--output-dir", type=Path, required=True)
args = p.parse_args(argv)
require(not args.output_dir.exists(), "Use a fresh output directory")
paths = {}
for item in args.run:
name, separator, path = item.partition("=")
require(separator and name and path and name not in paths, "Use unique --run NAME=PATH values")
paths[name] = Path(path)
report = build(args.protocol, args.cases, paths, args.reference,
args.splits.split(",") if args.splits is not None else None)
args.output_dir.mkdir(parents=True, exist_ok=False)
(args.output_dir / "summary.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
(args.output_dir / "summary.md").write_text(markdown(report))
print(json.dumps({"complete": True, "output": str(args.output_dir),
"runs": len(report["runs"]), "episodes_per_run": len(report["cohort_case_ids"])}))
if __name__ == "__main__":
main()

Xet Storage Details

Size:
17.5 kB
·
Xet hash:
ba44e0d43bd16406e209f0b59de891f6519873408e6f40e1be077c715a483622

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.