RealArDor/NanoJev-SimuNRGTrans / source /scripts /run_jev_parallel_cases.py
RealArDor's picture
download
raw
24.8 kB
#!/usr/bin/env python3
"""Run fixed Jev game cases in ordered shards with a shared budget ceiling.
The existing pipeline and HTTP worker are unchanged. Each child has four HTTP
slots, 16 environments, greedy epsilon .1 and seed 17. Journals retain separate
namespaces: an API receipt is identified by (shard, call ID). Existing journals
can be reused with a fresh --output and the same --journal-root/case file.
Only --max-active shards run concurrently; a failed shard retries in a fresh
attempt directory after 30 seconds without stopping other running shards.
"""
import argparse
from collections import Counter
from decimal import Decimal, ROUND_CEILING, ROUND_FLOOR
import hashlib
import json
import math
import os
from pathlib import Path
import random
import shutil
import signal
import subprocess
import sys
import time
from unified_game_pipeline import (
behavior_distribution, choose, digest, encode, file_digest, policy_request,
read_rows, source_hashes, summarize, validate_cases,
)
NANO = Decimal("0.000000001")
def write_json(path, value):
path = Path(path)
temporary = path.with_name(path.name + ".tmp")
temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")
os.replace(temporary, path)
def money(value):
if isinstance(value, bool):
raise ValueError("Boolean cost is invalid")
result = Decimal(str(value))
if not result.is_finite() or result < 0:
raise ValueError("Cost must be finite and nonnegative")
return result
def audit_journal(path):
"""Mirror worker accounting, including interrupted starts and unknown failures."""
path = Path(path)
raw = path.read_bytes() if path.exists() else b""
rows = [json.loads(line) for line in raw.splitlines() if line.strip()]
starts, successes, finished, failed_unknown = {}, {}, set(), set()
unresolved = set()
for row in rows:
identity, status = row["id"], row["status"]
if not isinstance(identity, str):
raise ValueError("Invalid journal call ID")
if status == "started":
if identity in starts or identity in finished:
raise ValueError("Duplicate or restarted journal call ID")
money(row.get("reservation_usd", .002))
starts[identity] = row
unresolved.add(identity)
elif status in ("succeeded", "failed"):
if identity in finished:
raise ValueError("Duplicate terminal journal receipt")
finished.add(identity)
unresolved.discard(identity)
if status == "succeeded":
money(row["cost_usd"])
successes[identity] = row
elif row.get("unknown_cost") is True:
failed_unknown.add(identity)
else:
raise ValueError("Unknown journal status")
unknown = unresolved | failed_unknown
spent = sum((money(r["cost_usd"]) for r in successes.values()), Decimal(0))
reserved = sum((money(starts.get(key, {}).get("reservation_usd", .002))
for key in unknown), Decimal(0))
report = {"journal_sha256": hashlib.sha256(raw).hexdigest(), "journal_bytes": len(raw),
"rows": len(rows), "started_calls": len(starts), "succeeded_calls": len(successes),
"failed_calls": sum(r["status"] == "failed" for r in rows),
"reported_cost_usd": str(spent), "unknown_reserved_usd": str(reserved),
"accounted_usd": str(spent + reserved), "unresolved_started_ids": sorted(unresolved),
"unknown_cost_ids": sorted(unknown),
"zero_reported_cost_calls": sum(money(r["cost_usd"]) == 0 for r in successes.values())}
return report, successes
def allocate_budgets(total, accounts):
"""Ceil old reservations and floor the total in integer nano-dollars."""
total = money(total)
if total <= 0 or total > 3:
raise ValueError("This runner permits a total budget above zero and at most USD 3")
units = int((total / NANO).to_integral_value(rounding=ROUND_FLOOR))
used = [int((money(x) / NANO).to_integral_value(rounding=ROUND_CEILING)) for x in accounts]
remaining = units - sum(used)
if not used or remaining < len(used) * 2_000_000:
raise ValueError("Insufficient budget after previous costs and unknown reservations")
each, extra = divmod(remaining, len(used))
budgets = [str(Decimal(v + each + int(i < extra)) * NANO) for i, v in enumerate(used)]
assert sum(map(Decimal, budgets)) <= total
return budgets
def ordered_shards(cases, workers):
if workers < 1 or not cases:
raise ValueError("Use a nonempty cohort and positive worker count")
width = (len(cases) + workers - 1) // workers
return [cases[i:i+width] for i in range(0, len(cases), width)]
def prepare_journals(root, prior, case_sha, shards):
"""Copy a stable old journal only to shard zero, or reuse all existing shards."""
root = Path(root)
declaration = {"schema": "nanojev-parallel-journals-v1", "cases_sha256": case_sha,
"shard_cases": [[c["id"] for c in group] for group in shards]}
marker = root / "parallel_journals.json"
if root.exists():
if prior:
raise ValueError("For journal reuse omit --prior-journal-dir; prior bytes are already present")
saved = json.loads(marker.read_text())
if any(saved.get(k) != v for k, v in declaration.items()):
raise ValueError("Existing journal root belongs to different cases or shards")
for i in range(len(shards)):
if not (root / f"shard_{i:02d}").is_dir():
raise ValueError("Existing journal root is missing a registered shard")
return saved
root.mkdir(parents=True)
for i in range(len(shards)):
(root / f"shard_{i:02d}").mkdir()
if prior:
source = Path(prior) / "calls.jsonl"
before, _ = audit_journal(source)
if not source.is_file():
raise ValueError("Prior journal file is missing")
destination = root / "shard_00/calls.jsonl"
shutil.copy2(source, destination)
if file_digest(source) != before["journal_sha256"] or file_digest(destination) != before["journal_sha256"]:
raise RuntimeError("Prior journal changed during copying; stop its writer before restarting")
declaration["prior_copy"] = {"destination_shard": 0, "source": str(source.resolve()), **before}
else:
declaration["prior_copy"] = None
write_json(marker, declaration)
return declaration
def child_command(args, cases, episodes, journal, budget):
command = [sys.executable, str(Path(__file__).with_name("unified_game_pipeline.py")), "rollout",
"--cases", str(cases), "--output", str(episodes), "--engine", "jev",
"--controller", "greedy", "--epsilon", "0.1", "--seed", "17",
"--env-batch", "16", "--batch-questions", "16", "--max-length", "8192",
"--env-file", str(args.env_file.resolve()), "--journal-dir", str(journal),
"--budget-usd", budget]
if args.remote_host:
command += ["--remote-host", args.remote_host, "--remote-command", args.remote_cmd]
return command
def validate_episode(episode, case, policy_id, calls):
if (episode["case"] != case or episode.get("complete") is not True or
episode.get("continuation_policy_id") != policy_id or type(episode.get("success")) is not bool):
raise ValueError("Incomplete episode, changed case or policy identity")
final = episode["final_info"]
if (final.get("terminated") is not True or final.get("truncated") or
final["success"] != episode["success"]):
raise ValueError("Episode does not have a real terminal outcome")
if case["spec"]["task"] == "shooting" and episode["success"] != (final["episode_metrics"]["kills"] > 0):
raise ValueError("Shooting success differs from its real kill count")
steps = episode["steps"]
rng = random.Random(int(digest([case["id"], 17])[:16], 16))
for i, step in enumerate(steps):
if step["truncated"] or step["terminated"] != (i == len(steps)-1):
raise ValueError("Early terminal, truncation or unfinished trajectory")
expected_behavior = behavior_distribution(step["scores"], "greedy", .1)
if step["behavior_probs"] != expected_behavior or step["action"] != choose(expected_behavior, rng):
raise ValueError("Recorded actions do not replay from the fixed controller")
if set(step["scores"]) != set(step["observation"]["candidates"]):
raise ValueError("Invalid action support")
if len(step["scores"]) == 1:
if not step.get("forced") or step["answers"]:
raise ValueError("A single-action transition must be recorded as forced")
continue
answer = step["answers"]["action"]
receipt = calls.get(answer["source_api_call_id"])
expected_input = {"model": "typesafe-ai/jev", "state": step["observation"]["state"],
"questions": policy_request(step["observation"], case["id"])["questions"]}
if (receipt is None or receipt["input"] != expected_input or
receipt["input_sha256"] != answer["source_input_sha256"] or
receipt["native_probs"]["action"] != answer["native_probabilities"]):
raise ValueError("Missing or mismatched per-shard successful API receipt")
native = answer["native_probabilities"]
if set(native) != set(step["scores"]) or any(not math.isfinite(p) or not 0 <= p <= 1 for p in native.values()):
raise ValueError("Invalid native distribution")
total = sum(native.values())
if total <= 0 or any(abs(step["scores"][k] - native[k] / total) > 1e-12 for k in native):
raise ValueError("Controller scores do not equal normalized native probabilities")
def merge_results(cases, case_path, jobs, output, expected_sources):
"""Only complete verified children may produce a complete merged artifact."""
output = Path(output)
expected_ids = [c["id"] for c in cases]
expected_cases = {c["id"]: c for c in cases}
rows, raw_by_id, source_map, child_receipts = {}, {}, {}, []
policy = None
for job in jobs:
if job.get("returncode") != 0:
raise ValueError("Every registered child must finish with exit code zero")
path = Path(job["episodes"])
manifest_path = path.with_suffix(".manifest.json")
manifest = json.loads(manifest_path.read_text())
group = read_rows(job["cases"])
if any(expected_cases.get(case["id"]) != case for case in group):
raise ValueError("Child case definitions differ from the original frozen file")
if (manifest.get("finished") is not True or manifest["episode_sha256"] != file_digest(path) or
manifest["cases_sha256"] != file_digest(job["cases"]) or
manifest["selected_cases"] != [c["id"] for c in group]):
raise ValueError("Child completion, case selection or source hash differs")
current = manifest["policy"]
if (current.get("engine") != "jev" or current.get("controller") != "greedy" or
current.get("epsilon") != .1 or current.get("sampling_seed") != 17 or
current.get("model") != "typesafe-ai/jev" or current.get("temperature") != 1.0 or
current.get("tie_break") != "lexicographic_first" or
current.get("environment_contract") != "finite_task_deadline_v1" or
current.get("source_sha256") != expected_sources or
manifest["continuation_policy_id"] != digest(current)):
raise ValueError("Child policy or implementation differs from the frozen contract")
if policy is not None and current != policy:
raise ValueError("Mixed child policies")
policy = current
snapshot = manifest_path.parent / manifest["source_snapshot"]
for name, sha in expected_sources.items():
if file_digest(snapshot / name) != sha:
raise ValueError("Child source snapshot differs")
accounting, calls = audit_journal(Path(job["journal"]) / "calls.jsonl")
if money(accounting["accounted_usd"]) > money(job["budget_usd"]) + NANO:
raise ValueError("Child reported costs and reservations exceed its allocation")
raw_lines = [line for line in path.read_bytes().splitlines(keepends=True) if line.strip()]
if len(raw_lines) != len(group):
raise ValueError("Missing or extra child episode")
child_rows = []
for case, raw in zip(group, raw_lines):
row = json.loads(raw)
validate_episode(row, case, manifest["continuation_policy_id"], calls)
if case["id"] in rows:
raise ValueError("Duplicate episode across shards")
rows[case["id"]], raw_by_id[case["id"]] = row, raw
source_map[case["id"]] = job["shard"]
child_rows.append(row)
if manifest["summary"] != summarize(child_rows):
raise ValueError("Child aggregate metrics differ from raw episodes")
child_receipts.append({"shard": job["shard"], "manifest_sha256": file_digest(manifest_path),
"episode_sha256": file_digest(path), "cases_sha256": file_digest(job["cases"]),
"manifest": str(manifest_path), "journal": job["journal"],
"budget_usd": job["budget_usd"], "accounting": accounting,
"attempts": job.get("attempts", [])})
if set(rows) != set(expected_ids) or len(rows) != len(cases):
raise ValueError("Merged collection must contain every original case exactly once")
destination = output / "episodes.jsonl"
if destination.exists():
raise ValueError("Merged output already exists")
ordered = [rows[key] for key in expected_ids]
temporary = destination.with_suffix(".jsonl.tmp")
with temporary.open("xb") as handle:
for key in expected_ids:
raw = raw_by_id[key]
handle.write(raw if raw.endswith(b"\n") else raw + b"\n")
os.replace(temporary, destination)
snapshot = output / "episodes.sources"
snapshot.mkdir()
for name, sha in expected_sources.items():
source = Path(__file__).with_name(name)
if file_digest(source) != sha:
raise ValueError("Local source changed during collection")
shutil.copy2(source, snapshot / name)
manifest = {"schema_version": "nanojev-unified-episodes-v1", "finished": True,
"policy": policy, "continuation_policy_id": digest(policy), "cases_sha256": file_digest(case_path),
"selected_cases": expected_ids, "episode_sha256": file_digest(destination),
"source_snapshot": snapshot.name, "summary": summarize(ordered),
"parallel_children": child_receipts, "episode_journal_shards": source_map,
"api_receipt_identity": "(journal shard, source_api_call_id); IDs are not globally unique",
"reported_cost_usd": str(sum(money(c["accounting"]["reported_cost_usd"]) for c in child_receipts)),
"unknown_reserved_usd": str(sum(money(c["accounting"]["unknown_reserved_usd"]) for c in child_receipts)),
"unknown_reservation_count": sum(len(c["accounting"]["unknown_cost_ids"]) for c in child_receipts),
"accounting_scope": "Includes prior journal exactly once in shard zero; zero reported cost is not independent billing verification."}
write_json(output / "episodes.manifest.json", manifest)
return manifest
def stop_children(processes):
alive = [p for p in processes if p.poll() is None]
for process in alive:
try:
os.killpg(process.pid, signal.SIGINT)
except ProcessLookupError:
pass
deadline = time.monotonic() + 5
while any(p.poll() is None for p in alive) and time.monotonic() < deadline:
time.sleep(.1)
for process in alive:
if process.poll() is None:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
def execute_jobs(jobs, max_active, max_attempts, update=lambda: None,
popen=subprocess.Popen, clock=time.monotonic, sleep=time.sleep):
"""Independent bounded retries; only external interruption stops other shards."""
if max_active < 1 or max_attempts < 1:
raise ValueError("Positive active and attempt limits are required")
running, eligible_after = {}, {}
origin = clock()
for job in jobs:
job.update(state="ready", attempts=[], returncode=None)
eligible_after[job["shard"]] = origin
try:
while True:
now = clock()
for shard, (job, process, log, attempt) in list(running.items()):
code = process.poll()
if code is None:
continue
log.close()
attempt.update(returncode=code, finished_elapsed_seconds=now-origin)
job["returncode"] = code
del running[shard]
if code == 0:
job["state"] = "succeeded"
elif len(job["attempts"]) >= max_attempts:
job["state"] = "exhausted"
else:
job["state"] = "cooldown"
eligible_after[shard] = now + 30.0
attempt["retry_delay_seconds"] = 30
if all(j["state"] in ("succeeded", "exhausted") for j in jobs):
update()
return all(j["state"] == "succeeded" for j in jobs)
available = [j for j in jobs if j["state"] in ("ready", "cooldown")
and eligible_after[j["shard"]] <= now]
# Fresh shards are not starved by a repeatedly failing early shard.
available.sort(key=lambda j: (len(j["attempts"]), j["shard"]))
for job in available[:max(0, max_active-len(running))]:
number = len(job["attempts"]) + 1
directory = Path(job["cases"]).parent / f"attempt_{number:02d}"
directory.mkdir(exist_ok=False)
episodes = directory / "episodes.jsonl"
command = list(job["command"])
command[command.index("--output") + 1] = str(episodes)
# Journal and USD allocation are identical for every attempt.
if (command[command.index("--journal-dir") + 1] != job["journal"] or
command[command.index("--budget-usd") + 1] != job["budget_usd"]):
raise ValueError("Attempt changed its journal or fixed budget")
log = (directory / "pipeline.log").open("x")
attempt = {"attempt": number, "episodes": str(episodes), "command": command,
"started_elapsed_seconds": clock()-origin, "returncode": None}
job["attempts"].append(attempt)
try:
process = popen(command, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
except BaseException:
log.close()
raise
attempt["pid"] = process.pid
job.update(episodes=str(episodes), command=command, pid=process.pid,
returncode=None, state="running")
running[job["shard"]] = (job, process, log, attempt)
update()
sleep(1)
except BaseException:
# Explicit interruption or local runner failure is distinct from an API
# child returning nonzero. The latter never signals unrelated children.
stop_children([entry[1] for entry in running.values()])
for job, process, log, attempt in running.values():
log.close()
code = process.poll()
attempt.update(returncode=code, finished_elapsed_seconds=clock()-origin)
job.update(returncode=code, state="interrupted")
update()
raise
def main():
parser = argparse.ArgumentParser(description=__doc__)
for name in ("cases", "output", "journal-root", "env-file"):
parser.add_argument("--" + name, type=Path, required=True)
parser.add_argument("--prior-journal-dir", type=Path)
parser.add_argument("--remote-host")
parser.add_argument("--remote-cmd", "--remote-command", dest="remote_cmd")
parser.add_argument("--workers", type=int, default=8)
parser.add_argument("--max-active", type=int, default=2,
help="Concurrent shards; each uses four HTTP slots")
parser.add_argument("--max-attempts", type=int, default=4,
help="Maximum attempts per shard, sharing its original journal and budget")
parser.add_argument("--total-budget-usd", default="3")
args = parser.parse_args()
if bool(args.remote_host) != bool(args.remote_cmd):
parser.error("Provide both remote host and command, or neither")
cases = read_rows(args.cases)
validate_cases(cases)
if not cases:
parser.error("The fixed case file cannot be empty")
if args.workers < 1 or args.workers > 8:
parser.error("Use one through eight independent pipeline children")
if not 1 <= args.max_active <= args.workers or not 1 <= args.max_attempts <= 4:
parser.error("Use max-active from one to workers and max-attempts from one to four")
if args.output.exists():
parser.error("Use a fresh --output directory")
args.output, args.journal_root = args.output.resolve(), args.journal_root.resolve()
groups = ordered_shards(cases, args.workers)
journals = prepare_journals(args.journal_root, args.prior_journal_dir, file_digest(args.cases), groups)
prior = [audit_journal(args.journal_root / f"shard_{i:02d}/calls.jsonl")[0] for i in range(len(groups))]
budgets = allocate_budgets(args.total_budget_usd, [r["accounted_usd"] for r in prior])
args.output.mkdir(parents=True)
sources = source_hashes()
jobs = []
for i, group in enumerate(groups):
directory = args.output / f"shard_{i:02d}"
directory.mkdir()
subset = directory / "cases.jsonl"
subset.write_text("".join(encode(c) + "\n" for c in group))
episodes, journal = directory / "episodes.jsonl", args.journal_root / f"shard_{i:02d}"
jobs.append({"shard": i, "cases": str(subset), "episodes": str(episodes), "journal": str(journal),
"budget_usd": budgets[i], "returncode": None,
"command": child_command(args, subset, episodes, journal, budgets[i])})
report = {"schema": "nanojev-parallel-jev-run-v1", "finished": False,
"runner_sha256": file_digest(__file__), "cases_sha256": file_digest(args.cases),
"total_budget_usd": str(money(args.total_budget_usd)), "allocated_budget_usd": str(sum(map(Decimal, budgets))),
"prior_accounting": prior, "journal_declaration": journals, "jobs": jobs,
"source_sha256": sources, "http_concurrency_per_child": 4, "env_batch_per_child": 16,
"max_active": args.max_active, "max_attempts_per_shard": args.max_attempts,
"retry_delay_seconds": 30, "failed_shard_does_not_interrupt_others": True,
"controller": {"mode": "greedy", "epsilon": .1, "seed": 17}}
report_path = args.output / "parallel_run.json"
write_json(report_path, report)
try:
completed = execute_jobs(jobs, args.max_active, args.max_attempts,
update=lambda: write_json(report_path, report))
if not completed:
raise RuntimeError("One or more shards exhausted their attempts; all other shards have finished")
merged = merge_results(cases, args.cases, jobs, args.output, sources)
report.update(finished=True, merged_episode_sha256=merged["episode_sha256"],
reported_cost_usd=merged["reported_cost_usd"], unknown_reserved_usd=merged["unknown_reserved_usd"])
except BaseException as exc:
report.update(finished=False, error_type=type(exc).__name__)
raise
finally:
# Parse after stopping writers; interrupted requests remain reserved.
try:
report["final_accounting"] = [audit_journal(Path(j["journal"]) / "calls.jsonl")[0] for j in jobs]
report["final_reported_cost_usd"] = str(sum(money(r["reported_cost_usd"]) for r in report["final_accounting"]))
report["final_unknown_reserved_usd"] = str(sum(money(r["unknown_reserved_usd"]) for r in report["final_accounting"]))
except Exception as exc:
report["accounting_error"] = type(exc).__name__
report["finished"] = False
write_json(report_path, report)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
24.8 kB
·
Xet hash:
22096beabec48e0af3c5b6362f22d76cbda952d5a9971dc2400793546f178249

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.