RealArDor/NanoJev-SimuNRGTrans / source /scripts /audit_appo_supervision_data.py
RealArDor's picture
download
raw
24.3 kB
#!/usr/bin/env python3
"""Read-only APPO data diagnostics; no simulator, model, network or training.
Verify the complete frozen collection and both exported views before reporting
action-policy statistics and exact student-observation ambiguity. Reused visible
observations are measured, not automatically classified as environment leakage.
"""
import argparse
from collections import Counter, defaultdict
from contextlib import ExitStack
import hashlib
import json
import math
from pathlib import Path
from appo_basic_data import training_rows, validate_episode
from unified_game_pipeline import SPLITS, digest, file_digest
def _unique_object(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError("Duplicate JSON key: " + key)
result[key] = value
return result
def _reject_constant(value):
raise ValueError("Nonfinite JSON value: " + value)
def decode(text):
return json.loads(text, object_pairs_hook=_unique_object, parse_constant=_reject_constant)
def load_json(path):
return decode(Path(path).read_text())
def assert_hash(value, label):
if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
raise ValueError("Missing or invalid SHA256: " + label)
return value
def read_next_row(handle):
for line in handle:
if line.strip():
return decode(line)
return None
def text_sha(value):
if not isinstance(value, str):
raise ValueError("The standard student state must remain its exact recorded text")
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def quantiles(values):
if not values:
return {"n": 0, "mean": None}
ordered = sorted(values)
output = {"n": len(values), "mean": math.fsum(values) / len(values)}
for name, fraction in (("min", 0), ("p05", .05), ("median", .5), ("p95", .95), ("max", 1)):
position = fraction * (len(ordered) - 1)
lo, hi = math.floor(position), math.ceil(position)
output[name] = ordered[lo] + (position - lo) * (ordered[hi] - ordered[lo])
return output
def new_bucket():
return {"episodes": 0, "successes": 0, "failures": 0, "decisions": 0,
"failure_decisions": 0, "action_not_argmax": 0, "tied_argmax_decisions": 0,
"argmax": Counter(), "actual_action": Counter(), "entropy": [], "max_probability": []}
def bucket_report(bucket):
n = bucket["decisions"]
return {key: bucket[key] for key in ("episodes", "successes", "failures", "decisions", "failure_decisions")} | {
"expert_argmax_counts": dict(sorted(bucket["argmax"].items())),
"expert_argmax_frequencies": {key: value / n for key, value in sorted(bucket["argmax"].items())} if n else {},
"actual_action_counts": dict(sorted(bucket["actual_action"].items())),
"action_not_argmax_count": bucket["action_not_argmax"],
"action_not_argmax_rate": bucket["action_not_argmax"] / n if n else None,
"tied_argmax_decisions": bucket["tied_argmax_decisions"],
"expert_entropy_nats": quantiles(bucket["entropy"]),
"expert_max_probability": quantiles(bucket["max_probability"])}
def observe_group(groups, key, action, split, episode_id):
group = groups.setdefault(key, {"labels": Counter(), "splits": set(), "episodes": set()})
group["labels"][action] += 1
group["splits"].add(split)
group["episodes"].add(episode_id)
def ambiguity_report(groups):
total = sum(sum(group["labels"].values()) for group in groups.values())
repeated = sum(sum(group["labels"].values()) > 1 for group in groups.values())
ambiguous = [(key, group) for key, group in groups.items() if len(group["labels"]) > 1]
ambiguous_rows = sum(sum(group["labels"].values()) for _, group in ambiguous)
minimum_errors = sum(sum(group["labels"].values()) - max(group["labels"].values()) for group in groups.values())
examples = sorted(ambiguous, key=lambda item: (-sum(item[1]["labels"].values()), item[0]))[:5]
return {"decisions": total, "unique_observations": len(groups), "repeated_observation_groups": repeated,
"ambiguous_groups": len(ambiguous),
"ambiguous_fraction_of_unique_groups": len(ambiguous) / len(groups) if groups else None,
"ambiguous_fraction_of_repeated_groups": len(ambiguous) / repeated if repeated else None,
"decisions_in_ambiguous_groups": ambiguous_rows,
"fraction_of_decisions_in_ambiguous_groups": ambiguous_rows / total if total else None,
"ambiguous_groups_with_multiple_episodes": sum(len(group["episodes"]) > 1 for _, group in ambiguous),
"minimum_empirical_argmax_errors_for_deterministic_observation_mapping": minimum_errors,
"minimum_empirical_error_rate": minimum_errors / total if total else None,
"largest_ambiguous_groups": [{"observation_sha256": key, "labels": dict(sorted(group["labels"].items())),
"episode_count": len(group["episodes"]), "splits": sorted(group["splits"])}
for key, group in examples]}
def overlap_report(groups):
"""One hash per episode reset; report rates over episodes and unique hashes."""
cross = {key: value for key, value in groups.items() if len(value) > 1}
by_split = {}
for split in SPLITS:
total = sum(value[split] for value in groups.values())
shared = sum(value[split] for value in cross.values())
by_split[split] = {"episodes": total, "unique_initial_hashes": sum(value[split] > 0 for value in groups.values()),
"episodes_with_a_cross_split_match": shared,
"cross_split_match_rate": shared / total if total else None}
return {"unique_initial_hashes": len(groups), "cross_split_hashes": len(cross), "by_split": by_split,
"pairwise_shared_unique_hashes": {a + "/" + b: sum(value[a] > 0 and value[b] > 0 for value in groups.values())
for i, a in enumerate(SPLITS) for b in SPLITS[i + 1:]}}
def load_attested_inputs(episodes_path, protocol_path):
manifest_path = episodes_path.with_suffix(".manifest.json")
manifest, protocol = load_json(manifest_path), load_json(protocol_path)
if manifest.get("finished") is not True or manifest.get("limit") != 0:
raise ValueError("Require a finished full collection with limit=0")
if manifest.get("episode_sha256") != file_digest(episodes_path):
raise ValueError("Episode file SHA256 differs from its manifest")
if manifest.get("config_sha256") != file_digest(protocol_path):
raise ValueError("Collection protocol SHA256 mismatch")
reference = Path(protocol["case_file"])
case_path = reference if reference.is_absolute() else Path(__file__).resolve().parents[1] / reference
if protocol["case_sha256"] != file_digest(case_path) or manifest.get("cases_sha256") != protocol["case_sha256"]:
raise ValueError("Frozen cohort file SHA256 mismatch")
with case_path.open() as handle:
cases = [decode(line) for line in handle if line.strip()]
if len({case["id"] for case in cases}) != len(cases):
raise ValueError("Duplicate cases in the frozen cohort")
if dict(Counter(case["split"] for case in cases)) != protocol["case_counts"]:
raise ValueError("Frozen cohort split counts differ from protocol")
if any(case["spec"].get("task") != "shooting" or case["spec"].get("scenario") != "basic" for case in cases):
raise ValueError("Only the complete frozen Basic cohort is supported")
for key in ("selected_episodes", "selected_before_limit", "episodes"):
if manifest.get(key) != len(cases):
raise ValueError("Incomplete full cohort: " + key)
if manifest.get("selected_cases_sha256") != digest(cases):
raise ValueError("Selected case identities/specifications differ from full cohort")
policy = manifest["policy"]
if (policy.get("engine") != "sample_factory_appo" or policy.get("render_adapter") != "mirrored_native"
or policy.get("controller") != "greedy" or policy.get("epsilon") != .1):
raise ValueError("Unexpected expert or behavior policy contract")
if manifest.get("continuation_policy_id") != digest(policy):
raise ValueError("Collection policy identity mismatch")
primary = protocol["primary"]
model = policy["model"]
for field, expected in (("model", primary["checkpoint_repository"]), ("revision", primary["checkpoint_revision"]),
("checkpoint_sha256", primary["checkpoint_sha256"])):
if model.get(field) != expected:
raise ValueError("Expert model provenance mismatch: " + field)
if policy.get("sampling_seed") != primary["sampling_seed"]:
raise ValueError("Expert behavior sampling seed mismatch")
return manifest, protocol, cases, {"episodes_sha256": manifest["episode_sha256"],
"manifest_sha256": file_digest(manifest_path), "protocol_sha256": file_digest(protocol_path),
"cases_sha256": file_digest(case_path)}
def audit(episodes_path, protocol_path):
episodes_path, protocol_path = Path(episodes_path), Path(protocol_path)
manifest, protocol, cases, hashes = load_attested_inputs(episodes_path, protocol_path)
root = episodes_path.parent
buckets = defaultdict(new_bucket)
state_groups, request_groups = {}, {}
split_state_groups, split_request_groups = defaultdict(dict), defaultdict(dict)
initial_text, initial_pixels, initial_requests = defaultdict(Counter), defaultdict(Counter), defaultdict(Counter)
initial_rnn_hashes, seen = set(), set()
row_counts, failure_rows = Counter(), Counter()
physical_ticks = 0
dataset_hashes = {}
with ExitStack() as stack:
views = {}
for arm in ("hard", "soft"):
path = root / arm / "manifest.json"
dm = load_json(path)
if dm.get("episode_sha256") != hashes["episodes_sha256"] or dm.get("max_states_per_episode") != 0:
raise ValueError("Exported view is not the complete recorded trajectory: " + arm)
if dm.get("all_successes_and_failures_retained") is not True:
raise ValueError("Exported view does not attest failure retention")
dataset_hashes[arm] = {"manifest_sha256": file_digest(path), "splits": {}}
for split in SPLITS:
path = root / arm / (split + ".jsonl")
actual = file_digest(path)
if dm.get("split_sha256", {}).get(split) != actual:
raise ValueError("Exported view split SHA256 mismatch: " + arm + "/" + split)
dataset_hashes[arm]["splits"][split] = actual
views[(arm, split)] = stack.enter_context(path.open())
source = stack.enter_context(episodes_path.open())
for expected_case in cases:
episode = read_next_row(source)
if episode is None or episode.get("case") != expected_case:
raise ValueError("Episode is absent or differs from the ordered full frozen cohort")
if episode.get("complete") is not True or type(episode.get("success")) is not bool:
raise ValueError("Every episode must have an explicit complete flag and Boolean outcome")
if episode["case"]["id"] in seen or episode.get("continuation_policy_id") != manifest["continuation_policy_id"]:
raise ValueError("Duplicate episode or mismatched collection policy")
seen.add(episode["case"]["id"])
validate_episode(episode, manifest["policy"]["sampling_seed"])
receipt = episode.get("standard_independent_replay", {})
if receipt.get("passed") is not True or receipt.get("decisions") != len(episode["steps"]):
raise ValueError("Missing complete standard-only replay receipt")
observations = [step["observation"] for step in episode["steps"]] + [episode["final_observation"]]
if receipt.get("standard_observation_sequence_sha256") != digest(observations):
raise ValueError("Standard-only observation sequence receipt mismatch")
ticks = episode["mirrored_physical_ticks"]
if receipt.get("physical_ticks") != ticks:
raise ValueError("Standard replay physical ticks mismatch")
physical_ticks += ticks
split, eid = expected_case["split"], expected_case["id"]
for name in ("all", split):
buckets[name]["episodes"] += 1
buckets[name]["successes" if episode["success"] else "failures"] += 1
for arm in ("hard", "soft"):
for expected_row in training_rows(episode, soft=arm == "soft"):
if read_next_row(views[(arm, split)]) != expected_row:
raise ValueError("Exported training row differs from its actual expert transition: " + arm + "/" + eid)
row_counts[(arm, split)] += 1
if not episode["success"]:
failure_rows[(arm, split)] += 1
for index, step in enumerate(episode["steps"]):
pixel = step["pixel_input"]
pixel_hash = assert_hash(pixel.get("native_frame_sha256"), "native expert frame")
resized_hash = assert_hash(pixel.get("resized_uint8_sha256"), "resized expert frame")
if (pixel.get("native_frame_shape_hwc") != [120, 160, 3]
or pixel.get("native_direct_resize_sha256") != resized_hash
or pixel.get("native_resize_equivalence_verified") is not True
or pixel.get("student_text_fed_to_expert") is not False):
raise ValueError("Expert native pixel provenance mismatch")
text = step["observation"]["state"]
if decode(text).get("screen_size") != [320, 240]:
raise ValueError("Student state is not the unchanged standard viewport")
state_hash = text_sha(text)
request_hash = digest({key: step["request"][key] for key in ("state", "questions")})
if index == 0:
initial_text[state_hash][split] += 1
initial_pixels[pixel_hash][split] += 1
initial_requests[request_hash][split] += 1
initial_rnn_hashes.add(assert_hash(pixel.get("rnn_before_sha256"), "initial recurrent state"))
label, probs = step["expert_argmax"], step["policy_probs"]
for groups, key in ((state_groups, state_hash), (request_groups, request_hash),
(split_state_groups[split], state_hash), (split_request_groups[split], request_hash)):
observe_group(groups, key, label, split, eid)
entropy = -math.fsum(p * math.log(p) for p in probs.values() if p > 0)
maximum = max(probs.values())
for name in ("all", split):
bucket = buckets[name]
bucket["decisions"] += 1
bucket["failure_decisions"] += not episode["success"]
bucket["action_not_argmax"] += step["action"] != label
bucket["tied_argmax_decisions"] += sum(p == maximum for p in probs.values()) > 1
bucket["argmax"][label] += 1
bucket["actual_action"][step["action"]] += 1
bucket["entropy"].append(entropy)
bucket["max_probability"].append(maximum)
if read_next_row(source) is not None or any(read_next_row(handle) is not None for handle in views.values()):
raise ValueError("Extra episodes or exported training rows outside the frozen collection")
decisions = buckets["all"]["decisions"]
if (manifest.get("decisions") != decisions or manifest.get("actual_expert_forward_calls") != decisions
or manifest.get("actual_recurrent_resets") != len(cases)
or manifest.get("mirrored_physical_ticks") != physical_ticks):
raise ValueError("Collection aggregate counters disagree with complete trajectories")
for split in SPLITS:
for key, expected in (("split_records", buckets[split]["decisions"]),
("split_episodes", buckets[split]["episodes"]),
("split_successes", buckets[split]["successes"])):
if manifest.get(key, {}).get(split, 0) != expected:
raise ValueError("Collection aggregate split counters disagree: " + key + "/" + split)
if len(initial_rnn_hashes) != 1:
raise ValueError("Episode initial recurrent-state hashes differ")
return {"schema_version": "nanojev-appo-data-audit-v1", "validated": True,
"source_hashes": hashes, "exported_dataset_hashes": dataset_hashes,
"audit_implementation_sha256": file_digest(__file__),
"collector_validator_sha256": file_digest(Path(__file__).with_name("appo_basic_data.py")),
"full_cohort": {"episodes": len(cases), "decisions": decisions, "physical_ticks": physical_ticks,
"complete": True, "episode_failures_retained": buckets["all"]["failures"],
"all_visited_states_retained_in_both_views": True,
"view_rows_by_split": {"/".join(key): n for key, n in sorted(row_counts.items())},
"view_failure_rows_by_split": {arm + "/" + split: failure_rows[(arm, split)]
for arm in ("hard", "soft") for split in SPLITS}},
"policy_statistics": {key: bucket_report(buckets[key]) for key in ("all", *SPLITS)},
"exact_state_text_ambiguity": {"all": ambiguity_report(state_groups),
"by_split": {split: ambiguity_report(split_state_groups[split]) for split in SPLITS}},
"full_student_request_ambiguity": {"all": ambiguity_report(request_groups),
"by_split": {split: ambiguity_report(split_request_groups[split]) for split in SPLITS}},
"initial_cross_split_reuse": {"standard_state_text": overlap_report(initial_text),
"full_student_request": overlap_report(initial_requests),
"expert_native_pixels": overlap_report(initial_pixels)},
"notes": ["Entropy uses natural logarithms; statistics are decision-weighted, not independent episode estimates.",
"The hard target is the original expert argmax; action_not_argmax measures actual epsilon behavior divergence, not all exploration draws.",
"Ambiguity is an empirical conflict at exactly repeated student inputs, not an estimate of generalization error. No observed conflict does not prove sufficient information.",
"The expert also receives recurrent pixel history; reset pixel reuse does not imply identical latent environment identity.",
"Cross-split visible-state/pixel matches are reported as observation reuse, not automatically as episode or latent-state leakage.",
"Stored replay/resize receipts are checked offline; no simulator or image reconstruction is executed by this audit."]}
def self_check():
import tempfile
groups = {}
for label, split, episode in (("left", "train", "a"), ("right", "train", "b"), ("left", "test", "c")):
observe_group(groups, "same-visible-state", label, split, episode)
report = ambiguity_report(groups)
assert report["ambiguous_groups"] == 1 and report["minimum_empirical_argmax_errors_for_deterministic_observation_mapping"] == 1
assert report["minimum_empirical_error_rate"] == 1 / 3
overlap = overlap_report({"shared": Counter(train=2, test=1), "unique": Counter(train=1)})
assert overlap["by_split"]["train"]["cross_split_match_rate"] == 2 / 3
assert overlap["pairwise_shared_unique_hashes"]["train/test"] == 1
assert quantiles([0., 1.])["median"] == .5
for bad in ('{"a":1,"a":2}', '{"a":NaN}'):
try:
decode(bad)
except ValueError:
pass
else:
raise AssertionError("Invalid JSON was accepted")
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
case = {"id": "integrity-fixture", "split": "train", "seed": 17,
"spec": {"task": "shooting", "scenario": "basic"}}
case_path, episode_path, protocol_path = [root / name for name in ("cases.jsonl", "episodes.jsonl", "protocol.json")]
case_path.write_text(json.dumps(case) + "\n")
episode_path.write_text("{}\n") # Only manifest attestation is tested here; never passed to trajectory diagnostics.
primary = {"checkpoint_repository": "fixture-model", "checkpoint_revision": "fixture-revision",
"checkpoint_sha256": "0" * 64, "sampling_seed": 17}
protocol = {"case_file": str(case_path), "case_sha256": file_digest(case_path),
"case_counts": {"train": 1}, "primary": primary}
protocol_path.write_text(json.dumps(protocol))
policy = {"engine": "sample_factory_appo", "render_adapter": "mirrored_native",
"controller": "greedy", "epsilon": .1, "sampling_seed": 17,
"model": {"model": primary["checkpoint_repository"], "revision": primary["checkpoint_revision"],
"checkpoint_sha256": primary["checkpoint_sha256"]}}
manifest = {"finished": True, "limit": 0, "episode_sha256": file_digest(episode_path),
"config_sha256": file_digest(protocol_path), "cases_sha256": file_digest(case_path),
"selected_episodes": 1, "selected_before_limit": 1, "episodes": 1,
"selected_cases_sha256": digest([case]), "policy": policy, "continuation_policy_id": digest(policy)}
path = episode_path.with_suffix(".manifest.json")
path.write_text(json.dumps(manifest))
load_attested_inputs(episode_path, protocol_path)
for change in ({"finished": False}, {"limit": 1}, {"selected_before_limit": 2},
{"selected_cases_sha256": "1" * 64}, {"episode_sha256": "2" * 64}):
path.write_text(json.dumps(manifest | change))
try:
load_attested_inputs(episode_path, protocol_path)
except ValueError:
pass
else:
raise AssertionError("Corrupt or partial collection manifest accepted")
return {"self_check": "passed", "checks": ["empirical ambiguity", "episode versus unique-hash reuse", "quantiles", "strict JSON", "manifest hash and full-cohort negative controls"]}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--episodes", type=Path)
parser.add_argument("--output", type=Path)
parser.add_argument("--protocol", type=Path, default=Path(__file__).resolve().parents[1] / "configs/appo_basic_supervision_v1.json")
parser.add_argument("--self-check", action="store_true")
args = parser.parse_args()
if args.self_check:
print(json.dumps(self_check()))
return
if not args.episodes or not args.output:
parser.error("--episodes and --output are required")
if args.output.exists():
parser.error("Use a fresh diagnostic output path")
result = audit(args.episodes, args.protocol)
args.output.parent.mkdir(parents=True, exist_ok=True)
with args.output.open("x") as handle:
json.dump(result, handle, indent=2, allow_nan=False)
handle.write("\n")
print(json.dumps({"validated": True, "episodes": result["full_cohort"]["episodes"],
"decisions": result["full_cohort"]["decisions"],
"failures_retained": result["full_cohort"]["episode_failures_retained"]}))
if __name__ == "__main__":
main()

Xet Storage Details

Size:
24.3 kB
·
Xet hash:
98cafe779ea7d3031a9f6650bcfeb6bf1de86d7a8d3ce92e008d9c0d0af46dbf

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.