Buckets:
| """Verified finite-horizon n-step targets for a frozen recorded behavior policy. | |
| The index is keyed by (training_row_id, question_id). Terminal entries contain | |
| terminal_target; bootstrap entries contain only a public request, recorded | |
| behavior probabilities, and action/question mapping. Gamma is one. Native game | |
| score rewards are not success rewards, and physical reposition moves do not add | |
| decision timesteps. This module performs no inference, simulation or API calls. | |
| """ | |
| from collections import Counter | |
| import copy | |
| import json | |
| import math | |
| from pathlib import Path | |
| import random | |
| from unified_game_pipeline import ( | |
| SPLITS, behavior_distribution, choose, digest, environment_group, | |
| file_digest, outcome_request, validate_cases, | |
| ) | |
| IDENTITY_TOLERANCE = 1e-9 | |
| def _require(condition, message): | |
| if not condition: | |
| raise ValueError(message) | |
| def _object(pairs): | |
| result = {} | |
| for key, value in pairs: | |
| _require(key not in result, f"Duplicate JSON key: {key}") | |
| result[key] = value | |
| return result | |
| def _nonfinite(value): | |
| raise ValueError(f"Nonfinite JSON value: {value}") | |
| def _parse(text): | |
| return json.loads(text, object_pairs_hook=_object, parse_constant=_nonfinite) | |
| def _probabilities(values, candidates, where): | |
| _require(isinstance(values, dict) and set(values) == set(candidates), f"{where}: probability candidate mismatch") | |
| _require(all(type(p) in (int, float) and math.isfinite(p) and 0 <= p <= 1 for p in values.values()), | |
| f"{where}: probabilities must be finite numbers in [0,1]") | |
| _require(abs(math.fsum(values.values()) - 1) <= IDENTITY_TOLERANCE, | |
| f"{where}: behavior probabilities must sum to one") | |
| def weighted_bootstrap(action_values, behavior_probs): | |
| """Return sum_a recorded_pi(a|s) Q_target(s,a); never maximize or refit pi.""" | |
| _require(isinstance(action_values, dict) and bool(action_values), "Nonempty action values are required") | |
| _require(all(type(v) in (int, float) and math.isfinite(v) and 0 <= v <= 1 for v in action_values.values()), | |
| "Success action values must be finite probabilities") | |
| _probabilities(behavior_probs, action_values, "bootstrap") | |
| result = math.fsum(behavior_probs[action] * action_values[action] for action in action_values) | |
| # A unit-sum distribution can exceed one by floating-point roundoff. Keep | |
| # the same tolerance as the probability validation; never repair bad pi. | |
| _require(-IDENTITY_TOLERANCE <= result <= 1 + IDENTITY_TOLERANCE, | |
| "Weighted success value is outside the probability range") | |
| return min(1.0, max(0.0, result)) | |
| def _verify_behavior(episode, declaration): | |
| case, steps = episode["case"], episode["steps"] | |
| rng = random.Random(int(digest([case["id"], declaration["sampling_seed"]])[:16], 16)) | |
| for index, step in enumerate(steps): | |
| where = f"{case['id']} decision {index}" | |
| obs = step["observation"] | |
| _require(set(obs) == {"task", "state", "candidates", "remaining_steps", "step"}, f"{where}: invalid observation schema") | |
| _require(obs["task"] == case["spec"]["task"] and isinstance(obs["state"], str) and obs["state"], | |
| f"{where}: invalid public task/state") | |
| candidates = obs["candidates"] | |
| _require(isinstance(candidates, dict) and bool(candidates) and | |
| all(isinstance(k, str) and k and isinstance(v, str) and v for k, v in candidates.items()), | |
| f"{where}: live observation needs described candidates") | |
| _require(type(obs["remaining_steps"]) is int and obs["remaining_steps"] > 0 and | |
| type(obs["step"]) is int and obs["step"] >= 0, f"{where}: invalid remaining/physical steps") | |
| _require(step["action"] in candidates, f"{where}: executed action is not a candidate") | |
| _probabilities(step.get("behavior_probs"), candidates, where) | |
| scores = step.get("scores") | |
| _require(isinstance(scores, dict) and set(scores) == set(candidates) and | |
| all(type(p) in (int, float) and math.isfinite(p) and 0 <= p <= 1 for p in scores.values()), | |
| f"{where}: invalid recorded scores") | |
| if len(candidates) == 1: | |
| _require(step.get("forced") is True and not step.get("answers"), f"{where}: singleton must be recorded as forced") | |
| else: | |
| _require(step.get("forced") is not True, f"{where}: multiple candidates cannot be forced") | |
| answers = step.get("answers", {}) | |
| if declaration["controller"] == "q_greedy": | |
| _require(set(answers) == {"success_" + a for a in candidates}, f"{where}: missing Q answers") | |
| for action in candidates: | |
| answer = answers["success_" + action] | |
| _require(answer.get("type") == "boolean", f"{where}: Q answer must be Boolean") | |
| probs = answer.get("probabilities", {}) | |
| _require(set(probs) == {"false", "true"} and all( | |
| type(p) in (int, float) and math.isfinite(p) and 0 <= p <= 1 for p in probs.values()), | |
| f"{where}: invalid Boolean output") | |
| # The persistent predictor exposes finite-precision FP32 softmax values. | |
| _require(abs(math.fsum(probs.values()) - 1) <= 1e-5, f"{where}: invalid Boolean probability sum") | |
| _require(abs(probs["true"] - scores[action]) <= IDENTITY_TOLERANCE, f"{where}: Q score differs from recorded answer") | |
| mode = "sample" if declaration["engine"] == "random" or declaration["controller"] == "sample" else "greedy" | |
| expected = behavior_distribution(scores, mode, declaration["epsilon"]) | |
| _require(all(abs(expected[a] - step["behavior_probs"][a]) <= IDENTITY_TOLERANCE for a in candidates), | |
| f"{where}: behavior probabilities differ from frozen controller") | |
| _require(choose(expected, rng) == step["action"], f"{where}: action differs from recorded policy RNG") | |
| def _load_episodes(path, dataset_manifest): | |
| path = Path(path) | |
| sidecar = path.with_suffix(".manifest.json") | |
| manifest = _parse(sidecar.read_text()) | |
| _require(manifest.get("schema_version") == "nanojev-unified-episodes-v1" and manifest.get("finished") is True, | |
| "TD requires a finished episode manifest") | |
| episode_sha = file_digest(path) | |
| _require(manifest.get("episode_sha256") == episode_sha == dataset_manifest.get("episode_sha256"), | |
| "Episode SHA256 does not match both manifests") | |
| declaration, policy_id = manifest["policy"], manifest["continuation_policy_id"] | |
| _require(policy_id == digest(declaration) == dataset_manifest.get("continuation_policy_id"), | |
| "Continuation policy ID does not match its frozen declaration and dataset") | |
| _require(declaration == dataset_manifest.get("collection_policy"), "Collection policy declarations disagree") | |
| _require(declaration.get("environment_contract") == "finite_task_deadline_v1" and | |
| declaration.get("tie_break") == "lexicographic_first", "Unsupported frozen environment/controller contract") | |
| _require(declaration.get("engine") in {"checkpoint", "jev", "random"} and | |
| declaration.get("controller") in {"greedy", "sample", "q_greedy"} and | |
| type(declaration.get("sampling_seed")) is int, "Invalid frozen policy declaration") | |
| _require(declaration.get("source_sha256") and all(isinstance(h, str) and len(h) == 64 | |
| for h in declaration["source_sha256"].values()), "Frozen policy must declare source hashes") | |
| snapshot_name = manifest.get("source_snapshot") | |
| _require(snapshot_name is None or isinstance(snapshot_name, str) and | |
| snapshot_name not in {"", ".", ".."} and Path(snapshot_name).name == snapshot_name, | |
| "Source snapshot must name a sibling directory") | |
| snapshot = path.parent / snapshot_name if snapshot_name else None | |
| snapshot_verified = False | |
| if snapshot is not None and snapshot.exists(): | |
| _require(snapshot.is_dir(), "Source snapshot must be a directory") | |
| for name, expected_sha in declaration["source_sha256"].items(): | |
| _require(Path(name).name == name and file_digest(snapshot / name) == expected_sha, | |
| f"Source snapshot hash mismatch: {name}") | |
| snapshot_verified = True | |
| episodes = [_parse(line) for line in path.read_text().splitlines() if line.strip()] | |
| _require(bool(episodes), "No episodes supplied") | |
| validate_cases([episode["case"] for episode in episodes]) | |
| by_id = {episode["case"]["id"]: episode for episode in episodes} | |
| selected = manifest.get("selected_cases") | |
| _require(isinstance(selected, list) and len(selected) == len(set(selected)) and set(selected) == set(by_id), | |
| "Selected case IDs do not exactly match episode rows") | |
| for episode in episodes: | |
| _require(episode.get("complete") is True and episode.get("continuation_policy_id") == policy_id, | |
| "Incomplete or mixed-policy episode") | |
| final, steps = episode.get("final_info", {}), episode.get("steps") | |
| success = episode.get("success") | |
| _require(type(success) is bool and final.get("success") is success and | |
| final.get("terminated") is True and final.get("truncated") is False, | |
| "Episode success/terminal information is inconsistent") | |
| _require(isinstance(steps, list), "Episode steps must be a list") | |
| for index, step in enumerate(steps): | |
| _require(step.get("truncated") is False and step.get("terminated") is (index == len(steps) - 1), | |
| "External truncation, early termination, or missing terminal transition") | |
| _require(step.get("info", {}).get("terminated") is step["terminated"] and | |
| step["info"].get("truncated") is False, "Transition terminal information disagrees") | |
| if steps: | |
| _require(steps[-1]["info"] == final, "Last transition information differs from final information") | |
| _verify_behavior(episode, declaration) | |
| return by_id, manifest, {"episodes_sha256": episode_sha, "manifest_sha256": file_digest(sidecar), | |
| "policy_declaration_sha256_verified": True, "source_snapshot_verified": snapshot_verified, | |
| "source_snapshot_note": "verified against declared hashes" if snapshot_verified else | |
| "snapshot unavailable; declaration identity verified, executable snapshot not reattested by this loader"} | |
| def load_td_index(path, records, dataset_manifest, n_step): | |
| """Return ({(row_id, qid): entry}, JSON audit) for all outcome rows/splits. | |
| For action a_t, n transitions cover decisions t..t+n-1. If the episode ends | |
| inside that interval, the target is its observed success. Otherwise bootstrap | |
| at the recorded pre-action state s_(t+n), averaging the frozen target network | |
| with that state's recorded behavior distribution. Neither realized success nor | |
| post-bootstrap observations are placed in a nonterminal network request. | |
| """ | |
| _require(type(n_step) is int and n_step > 0, "n_step must be a positive integer") | |
| _require(dataset_manifest.get("max_states_per_episode") == 0, "TD requires all transitions, without outcome thinning") | |
| episodes, manifest, provenance = _load_episodes(path, dataset_manifest) | |
| policy_id = manifest["continuation_policy_id"] | |
| group_ids = {cid: environment_group(episode["case"]) for cid, episode in episodes.items()} | |
| expected = {(cid, index) for cid, episode in episodes.items() for index in range(len(episode["steps"]))} | |
| seen, row_ids, group_splits, index, counts = set(), set(), {}, {}, Counter() | |
| ignored_policy = 0 | |
| for row in records: | |
| _require(isinstance(row.get("id"), str) and row["id"] and row["id"] not in row_ids, "Duplicate or invalid training row ID") | |
| row_ids.add(row["id"]) | |
| meta = row.get("metadata", {}) | |
| role, split = meta.get("record_role"), row.get("split") | |
| _require(role in {"policy", "outcome"} and split in SPLITS, "Invalid row role or split") | |
| group = meta.get("source_group_id") | |
| _require(isinstance(group, str) and group and (group not in group_splits or group_splits[group] == split), | |
| "A source environment group crosses splits") | |
| group_splits[group] = split | |
| if role == "policy": | |
| ignored_policy += 1 | |
| continue | |
| key = meta.get("episode_id"), meta.get("decision_index") | |
| _require(type(key[1]) is int and key in expected and key not in seen, "Missing, duplicate or unknown outcome transition") | |
| seen.add(key) | |
| episode = episodes[key[0]] | |
| case, t = episode["case"], key[1] | |
| step, steps = episode["steps"][t], episode["steps"] | |
| obs, action = step["observation"], step["action"] | |
| _require(split == case["split"] and group == group_ids[key[0]] and meta.get("task") == case["spec"]["task"], | |
| "Outcome split/task/source group disagrees with its trajectory") | |
| _require(meta.get("continuation_policy_id") == policy_id and meta.get("spec") == case["spec"], | |
| "Outcome policy/spec disagrees with its trajectory") | |
| _require(meta.get("executed_action") == meta.get("conditioned_action") == action, | |
| "Outcome question is not bound to the executed action") | |
| _require(row.get("state") == obs["state"] and meta.get("remaining_steps") == obs["remaining_steps"] and | |
| meta.get("public_observation_sha256") == digest(obs["state"]) and | |
| row.get("state_id") == digest([group, case["seed"], t, obs["state"]]), | |
| "Outcome state/remaining steps/identity disagrees with its trajectory") | |
| request = outcome_request(obs, row["id"], [action]) | |
| _require(row.get("questions") == request["questions"], "Outcome question text differs from its recorded action") | |
| qid = "success_" + action | |
| _require(row.get("gold") == {qid: episode["success"]} and type(row["gold"][qid]) is bool and | |
| row.get("gold_label_kind") == {qid: "observed_outcome"} and "gold_probs" not in row and | |
| meta.get("episode_success") is episode["success"], "Outcome label is not the actual terminal success") | |
| remaining = len(steps) - t | |
| entry = {"episode_id": key[0], "decision_index": t, "split": split, | |
| "n_step": n_step, "actual_decision_steps": min(n_step, remaining), "gamma": 1.0, | |
| "continuation_policy_id": policy_id} | |
| if remaining <= n_step: | |
| entry["terminal_target"] = float(episode["success"]) | |
| counts[(split, "terminal")] += 1 | |
| else: | |
| future = steps[t + n_step] | |
| future_obs = future["observation"] | |
| request_id = "td-bootstrap:" + digest([policy_id, key[0], t + n_step]) | |
| entry.update(bootstrap_decision_index=t + n_step, | |
| request=copy.deepcopy(outcome_request(future_obs, request_id)), | |
| behavior_probs=copy.deepcopy(future["behavior_probs"]), | |
| action_to_qid={action: "success_" + action for action in future_obs["candidates"]}) | |
| counts[(split, "bootstrap")] += 1 | |
| index[(row["id"], qid)] = entry | |
| _require(seen == expected, f"Outcome rows do not cover every transition: missing {len(expected-seen)}") | |
| audit = {**provenance, "episodes": len(episodes), "outcome_rows": len(index), "ignored_policy_rows": ignored_policy, | |
| "n_step": n_step, "gamma": 1.0, "continuation_policy_id": policy_id, | |
| "index_key": "(row_id, question_id)", "decision_timestep": "recorded model/controller action; excludes internal physical reposition moves", | |
| "target": "terminal success indicator, otherwise frozen-target action values averaged with recorded behavior_probs", | |
| "native_game_rewards_used": False, "behavior_distribution_and_rng_verified": True, | |
| "cross_split_environment_groups": 0, "complete_transition_coverage": True, | |
| "by_split": {split: {kind: counts[(split, kind)] for kind in ("terminal", "bootstrap")} for split in SPLITS}} | |
| return index, audit | |
Xet Storage Details
- Size:
- 16.2 kB
- Xet hash:
- f07109e2327c2393b214c7e496c9670aa77925135644027b9afb2ddf53d5b501
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.