Buckets:
| #!/usr/bin/env python3 | |
| """Build a separate three-system replay with a real native-Qwen maze run. | |
| The original arcade and media are read-only inputs. All selected source episodes | |
| are replayed on CPU; no API call, model forward, or episode search is performed. | |
| """ | |
| import argparse | |
| import copy | |
| import hashlib | |
| import json | |
| import math | |
| from pathlib import Path | |
| import build_arcade_demo as arcade | |
| import evaluate_model_edges_maze as maze | |
| from evaluate_composed_maze import digest, local_questions | |
| ORDER = ("jev", "nanojev", "base") | |
| QWEN_REVISION = "c1899de289a04d12100db370d81485cdf75e47ca" | |
| def require(condition, message): | |
| if not condition: | |
| raise ValueError(message) | |
| def frame_hash(system): | |
| raw = json.dumps(system["frames"], ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() | |
| return hashlib.sha256(raw).hexdigest() | |
| def load_cohort(path): | |
| rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] | |
| require(len({row["id"] for row in rows}) == len(rows), "Duplicate frozen cohort IDs") | |
| return {row["id"]: row for row in rows} | |
| def native_maze(path, frozen, controller_protocol, input_path, original_cohort_sha): | |
| source, episode, receipt = arcade.read_source(path, arcade.MAZE_CASE) | |
| require(source["schema"] == "nanojev-model-edges-maze-v1", "Unsupported native maze schema") | |
| require(source["model_role"] == "model_guided_local_edge_exploration", "Native maze control authority differs") | |
| require(source["source_episodes_sha256"] == arcade.sha_file(input_path), "Native input episode file SHA differs") | |
| require(source["source_cohort_sha256"] == [original_cohort_sha], "Native input does not reference the frozen original cohort") | |
| require(source["selected_episode_ids"] == [arcade.MAZE_CASE] and len(source["episodes"]) == 1, | |
| "Native run must preserve the explicitly selected single case") | |
| model = source["model"] | |
| require(model.get("engine") == "native" and model.get("model") == "Qwen/Qwen3-0.6B" and | |
| model.get("revision") == QWEN_REVISION and model.get("backend") == "untrained_qwen_lm_option_logits", | |
| "The maze baseline must be the recorded original Qwen model, not a trained checkpoint") | |
| require(source["implementation_sha256"] == arcade.sha_file(Path(maze.__file__)), "Frozen maze implementation SHA differs") | |
| renderer = Path(maze.__file__).with_name("evaluate_composed_maze.py") | |
| require(source["local_renderer_sha256"] == arcade.sha_file(renderer), "Native maze renderer SHA differs") | |
| runner = Path(source["native_runner"]) | |
| require(runner.resolve().parent == Path(__file__).resolve().parent and runner.suffix == ".py", "Native runner is outside the scripts directory") | |
| require(source["native_runner_sha256"] == arcade.sha_file(runner), "Native wrapper SHA differs") | |
| predictor = Path(maze.__file__).with_name("evaluate_native_qwen_navigation.py") | |
| require(model.get("native_predictor_sha256") == arcade.sha_file(predictor), "Native LM predictor SHA differs") | |
| require(model.get("decision_adapter") == "boolean_as_native_option_choice" and model.get("token_ids") == {"A": 32, "B": 33}, | |
| "Native Boolean adapter or vocabulary-token mapping differs") | |
| require(source["execution"].get("autoregressive_decode_steps") == 0 and source["execution"].get("generated_tokens") == 0, | |
| "Native maze report includes generated answer tokens") | |
| require(source["protocol"] == controller_protocol, "Native and trained maze controllers differ") | |
| require(source["question_template"] == local_questions(), "Native maze question template differs") | |
| initial = copy.deepcopy(episode["initial_state"]) | |
| require(initial == frozen["initial_state"], "Native maze does not start from the frozen case") | |
| require(episode["horizon"] == 2 * initial["size"] ** 2 and source["protocol"]["max_steps"] == 0, | |
| "Native maze must use the same complete attempt budget") | |
| env = maze.MazeEnvironment(initial, source["protocol"]["window_size"]) | |
| policy = maze.EdgeExplorer(**env.public_coordinates()) | |
| observations = {} | |
| native_receipts = 0 | |
| for row in episode["observations"]: | |
| coordinate = tuple(row["position"]) | |
| require(coordinate not in observations, "Repeated native prediction for a static maze node") | |
| observations[coordinate] = row | |
| native_receipts += verify_native_probabilities(row) | |
| consumed, collisions = set(), 0 | |
| frames = [{"position": initial["position"], "score": 0, "action": None, "probabilities": {}, | |
| "collision": False, "decision_source": "initial", "done": env.reached_goal()}] | |
| for index, step in enumerate(episode["steps"]): | |
| require(index < episode["horizon"] and not env.reached_goal(), "Native maze continues after its stopping condition") | |
| require(step["step_index"] == index, "Native maze step ordering differs") | |
| coordinate = tuple(policy.position) | |
| require(coordinate in observations, "Native maze action lacks a recorded prediction") | |
| observed = observations[coordinate] | |
| values = arcade.probabilities(observed["probabilities"], maze.DIRECTIONS, normalized=False) | |
| request = {"id": observed["id"], **env.observe()} | |
| require(observed["local_state"] == request["state"] and observed["input_sha256"] == digest(request), | |
| "Native visible request does not match the simulated state") | |
| if policy.needs_prediction(): | |
| require(coordinate not in consumed, "Native maze observation is consumed twice") | |
| policy.remember_prediction(values) | |
| consumed.add(coordinate) | |
| decision = policy.choose() | |
| require(decision is not None and all(step[key] == value for key, value in decision.items()), | |
| "Native maze controller decision differs") | |
| feedback = env.attempt(step["action"]) | |
| require(all(step[key] == value for key, value in feedback.items()), "Native maze physical feedback differs") | |
| policy.observe_transition(step["action"], feedback["next_position"], feedback["collision"]) | |
| collisions += int(feedback["collision"]) | |
| frames.append({"position": feedback["next_position"], "decision_position": list(coordinate), | |
| "score": int(feedback["goal_reached"]), "action": step["action"], "probabilities": values, | |
| "collision": feedback["collision"], "decision_source": step["mode"], | |
| "done": feedback["goal_reached"] or index + 1 == len(episode["steps"])}) | |
| require(consumed == set(observations), "Native maze has unused or missing recorded predictions") | |
| require(len(episode["steps"]) == episode["attempts"] and collisions == episode["collisions"], "Native maze counts differ") | |
| if env.reached_goal(): | |
| outcome = "goal" | |
| elif len(episode["steps"]) == episode["horizon"]: | |
| outcome = "horizon_exhausted" | |
| else: | |
| require(policy.choose() is None, "Native maze is truncated before a stopping condition") | |
| outcome = "frontier_exhausted" | |
| require(episode["status"] == outcome and episode["goal_completion"] == env.reached_goal(), "Native maze final outcome differs") | |
| system = {"id": "base", "name": "Untuned Qwen", "detail": "Native Qwen safety probabilities + the same verified-edge exploration", | |
| "color": arcade.COLORS["base"], "probability_kind": "independent_safety", | |
| "summary": {"steps": len(episode["steps"]), "score": int(env.reached_goal()), "collisions": collisions, "outcome": outcome}, | |
| "frames": frames} | |
| receipt.update(system_id="base", role=system["detail"], model=arcade.model_identity(source), | |
| model_adapter=model.get("decision_adapter"), policy="Deterministic exploration", | |
| runner_sha256=source["implementation_sha256"], native_runner=str(runner), native_runner_sha256=source["native_runner_sha256"], | |
| native_predictor_sha256=model["native_predictor_sha256"], | |
| original_weight_files_sha256_as_recorded=model.get("original_weight_files_sha256"), | |
| renderer_sha256=source["local_renderer_sha256"], source_episode_file_sha256=source["source_episodes_sha256"], | |
| original_cohort_sha256_as_recorded=source.get("source_cohort_sha256"), | |
| initial_state_sha256=digest(initial), frame_json_sha256=frame_hash(system), | |
| verified_transitions=len(episode["steps"]), recorded_nodes=len(consumed), final_outcome=outcome, | |
| native_receipts_verified=native_receipts, native_softmax_probability_tolerance=2e-6, | |
| replay="Exact visible requests, policy choices, physical feedback and final stopping condition") | |
| return system, receipt | |
| def verify_native_probabilities(observed): | |
| from evaluate_native_qwen_navigation import build_prompt | |
| receipts = observed["primitive_receipts"] | |
| require(set(receipts) == set(local_questions()), "Native receipt coverage differs") | |
| for qid, question in local_questions().items(): | |
| receipt = receipts[qid] | |
| require(receipt.get("backend") == "untrained_qwen_lm_option_logits" and | |
| receipt.get("decision_adapter") == "boolean_as_native_option_choice", "Native receipt backend differs") | |
| require(receipt["candidate_to_token"] == {"false": {"text": "A", "id": 32}, "true": {"text": "B", "id": 33}}, | |
| "Native false/true option mapping differs") | |
| logits = receipt["native_option_logits"] | |
| unconditional = receipt["native_option_unconditional_probs"] | |
| require(set(logits) == set(unconditional) == {"false", "true"}, "Native receipt options differ") | |
| require(all(type(value) in (int, float) and math.isfinite(value) for value in logits.values()), "Invalid native logits") | |
| require(all(type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 1 for value in unconditional.values()), | |
| "Invalid native vocabulary probability") | |
| offset = max(logits.values()) | |
| weights = {key: math.exp(value - offset) for key, value in logits.items()} | |
| p_true = weights["true"] / math.fsum(weights.values()) | |
| mass = math.fsum(unconditional.values()) | |
| require(0 < mass <= 1 + 1e-5 and math.isclose(mass, receipt["offered_token_mass"], rel_tol=1e-12, abs_tol=1e-12), | |
| "Native offered-token mass differs") | |
| observed_p = observed["probabilities"][qid.removeprefix("clear_")] | |
| require(abs(observed_p - p_true) <= 2e-6 and abs(observed_p - unconditional["true"] / mass) <= 2e-6, | |
| "Recorded safety probability differs from native conditional logits") | |
| public = {"state": observed["local_state"], "questions": {"action": {"type": "choice", | |
| "instructions": question["instructions"], "criteria": {key: question["criteria"][key] for key in ("false", "true")}}}} | |
| prompt, _ = build_prompt(public) | |
| require(hashlib.sha256(prompt.encode()).hexdigest() == receipt["prompt_sha256"], "Native complete prompt hash differs") | |
| require(type(receipt["prompt_tokens"]) is int and receipt["prompt_tokens"] > 0, "Invalid recorded prompt length") | |
| return len(receipts) | |
| def build(args): | |
| original_bytes = args.arcade_data.read_bytes() | |
| original = json.loads(original_bytes) | |
| manifest = json.loads(args.arcade_manifest.read_text(encoding="utf-8")) | |
| require(original["schema"] == "nanojev-arcade-v1", "Unsupported original arcade format") | |
| require(manifest["output"]["sha256"] == hashlib.sha256(original_bytes).hexdigest(), "Original arcade data hash differs") | |
| require(manifest["builder_sha256"] == arcade.sha_file(Path(arcade.__file__)), "Original arcade builder changed") | |
| maze_cohort, snake_cohort = load_cohort(args.maze_cohort), load_cohort(args.snake_cohort) | |
| native_cohort = load_cohort(args.native_maze_cohort) | |
| require(set(native_cohort) == {arcade.MAZE_CASE} and | |
| native_cohort[arcade.MAZE_CASE]["initial_state"] == maze_cohort[arcade.MAZE_CASE]["initial_state"], | |
| "Native input is not the same frozen 50x50 initial state") | |
| require(native_cohort[arcade.MAZE_CASE]["source_cohort_sha256"] == arcade.sha_file(args.maze_cohort), | |
| "Native single-case provenance differs from the original cohort") | |
| examples, audits, protected = [], [], {args.arcade_data.resolve(), args.arcade_manifest.resolve(), args.maze_native.resolve(), | |
| args.maze_cohort.resolve(), args.native_maze_cohort.resolve(), | |
| args.snake_cohort.resolve(), Path(arcade.__file__).resolve()} | |
| for old in original["examples"]: | |
| matches = [item for item in manifest["examples"] if item["case_id"] == old["id"]] | |
| require(len(matches) == 1, "Original arcade manifest case mismatch") | |
| old_audit = matches[0] | |
| require(len(old["systems"]) == len(old_audit["sources"]) == 3, "Expected three recorded systems") | |
| require({system["id"] for system in old["systems"]} == set(ORDER), "Unknown recorded system identity") | |
| example = copy.deepcopy(old) | |
| systems, sources, initial = {}, {}, None | |
| for system, source in zip(old["systems"], old_audit["sources"]): | |
| identity, path = system["id"], Path(source["path"]) | |
| protected.add(path.resolve()) | |
| require(arcade.sha_file(path) == source["sha256"], "An original source report changed") | |
| if old["game"] == "maze" and identity == "base": | |
| continue | |
| if old["game"] == "maze": | |
| state, fresh, receipt, protocol = arcade.maze_system(path, identity, system["name"], system["detail"]) | |
| frozen = maze_cohort[old["id"]] | |
| elif old["game"] == "snake": | |
| state, fresh, receipt, protocol = arcade.snake_system(path, old["id"], identity, system["name"], system["detail"]) | |
| frozen = snake_cohort[old["id"]] | |
| else: | |
| raise ValueError("Unsupported original game") | |
| require(state == frozen["initial_state"], "A source differs from the frozen initial state") | |
| require(fresh == system, "Replayed frames differ from the retained original arcade system") | |
| require(protocol == old_audit["controller"], "A retained controller differs") | |
| if initial is None: | |
| initial = state | |
| require(initial == state, "Compared systems have different initial states") | |
| systems[identity] = copy.deepcopy(system) | |
| sources[identity] = {**receipt, "system_id": identity, "initial_state_sha256": digest(state), | |
| "frame_json_sha256": frame_hash(system), "frames_unchanged_from_original_arcade": True, | |
| "final_outcome": system["summary"]["outcome"]} | |
| if old["game"] == "maze": | |
| require(old["id"] == arcade.MAZE_CASE, "The fixed maze case changed") | |
| native, native_receipt = native_maze(args.maze_native, maze_cohort[old["id"]], old_audit["controller"], | |
| args.native_maze_cohort, arcade.sha_file(args.maze_cohort)) | |
| require({key: initial[key] for key in ("walls", "position", "goal")} == old["initial"], "Maze public initial state differs") | |
| systems["base"], sources["base"] = native, native_receipt | |
| role = "All three models provide independent local safety probabilities. The same code explores and remembers verified edges; the baseline is original Qwen, not Starting NanoJev." | |
| else: | |
| require({key: initial[key] for key in ("body", "food")} == old["initial"], "Snake public initial state differs") | |
| role = old_audit["role_boundary"] | |
| example["systems"] = [systems[identity] for identity in ORDER] | |
| examples.append(example) | |
| audits.append({"case_id": old["id"], "game": old["game"], "initial_state_sha256": digest(initial), | |
| "controller": old_audit["controller"], "role_boundary": role, | |
| "probabilities": old_audit["probabilities"], "selection": old_audit["selection"], | |
| "sources": [sources[identity] for identity in ORDER]}) | |
| require({item["game"] for item in examples} == {"maze", "snake"} and len(examples) == 2, "Expected one maze and one Snake case") | |
| return {"schema": "nanojev-arcade-v1", "examples": examples}, audits, manifest, protected | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--arcade-data", type=Path, default=Path("web/arcade_results.json")) | |
| parser.add_argument("--arcade-manifest", type=Path, default=Path("assets/arcade_data_manifest.json")) | |
| parser.add_argument("--maze-native", type=Path, default=Path("results/model_edges_native.json")) | |
| parser.add_argument("--maze-cohort", type=Path, default=Path("results/rollout_pilot_episodes.jsonl")) | |
| parser.add_argument("--native-maze-cohort", type=Path, default=Path("results/side_by_side_maze_episode.jsonl")) | |
| parser.add_argument("--snake-cohort", type=Path, default=Path("results/arcade_snake_cohort.jsonl")) | |
| parser.add_argument("--output", type=Path, default=Path("web/side_by_side_results.json")) | |
| parser.add_argument("--manifest", type=Path, default=Path("assets/side_by_side_data_manifest.json")) | |
| parser.add_argument("--overwrite", action="store_true") | |
| args = parser.parse_args() | |
| for path in (args.output, args.manifest): | |
| require(args.overwrite or not path.exists(), f"Output exists; pass --overwrite: {path}") | |
| require(args.output.resolve() != args.manifest.resolve(), "Output and manifest paths must differ") | |
| before = {path: arcade.sha_file(path) for path in (args.arcade_data, args.arcade_manifest)} | |
| output, audits, original_manifest, protected = build(args) | |
| require(not {args.output.resolve(), args.manifest.resolve()} & protected, "An output would overwrite an input") | |
| raw = (json.dumps(output, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + "\n").encode() | |
| manifest = {"schema": "nanojev-side-by-side-data-v1", "builder_sha256": arcade.sha_file(Path(__file__)), | |
| "reused_arcade_builder_sha256": arcade.sha_file(Path(arcade.__file__)), | |
| "output": {"path": str(args.output), "sha256": hashlib.sha256(raw).hexdigest()}, | |
| "original_arcade": {"data_path": str(args.arcade_data), "data_sha256": before[args.arcade_data], | |
| "manifest_path": str(args.arcade_manifest), "manifest_sha256": before[args.arcade_manifest]}, | |
| "cohorts": {str(path): arcade.sha_file(path) for path in (args.maze_cohort, args.native_maze_cohort, args.snake_cohort)}, | |
| "system_order": list(ORDER), "examples": audits, "selection_protocol": original_manifest["selection_protocol"], | |
| "frame_semantics": original_manifest["frame_semantics"], "playback": original_manifest["playback"], | |
| "verification": {"all_selected_source_transitions_replayed": True, | |
| "retained_systems_have_identical_serialized_frames": 5, | |
| "maze_baseline_replaced_by_new_native_measurement": True, | |
| "verified_transitions": sum(source["verified_transitions"] for item in audits for source in item["sources"]), | |
| "original_arcade_files_unchanged": True, "new_model_calls": 0, "new_api_calls": 0}} | |
| for path, expected in before.items(): | |
| require(arcade.sha_file(path) == expected, "Original arcade changed during this build") | |
| for path in (args.output, args.manifest): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_bytes(raw) | |
| args.manifest.write_text(json.dumps(manifest, indent=2, ensure_ascii=False, allow_nan=False) + "\n", encoding="utf-8") | |
| print(json.dumps({"output": str(args.output), "examples": len(output["examples"]), **manifest["verification"]})) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 20.2 kB
- Xet hash:
- 0f8f223f9ad2f131484d4d68ef373b5e78f04998368dc903553e5808aab04479
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.