File size: 7,161 Bytes
9d6c005 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | """Validate the explicitly scoped research release, not production maturity."""
from pathlib import Path
import argparse,csv,hashlib,json,platform,re,subprocess,sys
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT))
import numpy as np
def load(name):return json.loads((ROOT/name).read_text())
def validate(output="validation_reproduced.json",require_replay=False):
gates=[]
def gate(name,passed,details):
gates.append({"name":name,"passed":bool(passed),"details":details})
required=["README.md","LICENSE","CITATION.cff","MANUSCRIPT.md","AUREOLE_R_v2.0.0_Research_Release.pdf",
"STATUS.json","CLAIMS.json","AI_AGENT_INDEX.json","MODEL_CARD.md","models/visibility_prior.npz",
"docs/ESTIMATOR_CONTRACT.md","docs/RELEASE_SCOPE.md","docs/RESIDUAL_THEORY.md",
"experiments.json","experiments_followup.json","scripts/demo.py","scripts/publish_hf.py",
"results/rendering_report.json","results/followup_report.json","results/training.json"]
missing=[p for p in required if not (ROOT/p).is_file()]
gate("required_release_files",not missing,{"missing":missing})
counts=[]
for start,expected in [("tests",28),("legacy/code",14)]:
result=subprocess.run([sys.executable,"-m","unittest","discover","-s",start,"-p","test_*.py","-v"],cwd=ROOT,capture_output=True,text=True)
match=re.search(r"Ran (\d+) tests?",result.stderr)
count=int(match.group(1)) if match else 0;counts.append(count)
gate(f"executable_tests_{start}",result.returncode==0 and count==expected,{"exit_code":result.returncode,"tests":count,"expected_tests":expected})
training=load("results/training.json");a=load("results/rendering_report.json");b=load("results/followup_report.json")
train=set(training["training_scene_ids"]);val=set(training["validation_scene_ids"]);test=set(training["test_scene_ids"])
follow=set(b["protocol"]["scene_ids"])
gate("independent_scene_splits",not(train&val or train&test or val&test or follow&(train|val|test)),
{"training_scenes":len(train),"validation_scenes":len(val),"first_test_scenes":len(test),"followup_scenes":len(follow)})
from aureole.renderer import VisibilityPrior,Scene,receiver_grid,light_grid
prior=VisibilityPrior(ROOT/"models/visibility_prior.npz")
pred=prior(Scene.create(900).features(receiver_grid(4,4)[:,None,:],light_grid(2)[None,:,:]))
gate("portable_model",np.isfinite(pred).all() and ((pred>=0)&(pred<=1)).all() and training["numpy_torch_max_abs_error"]<1e-5,
{"parameters":training["parameters"],"numpy_torch_max_abs_error":training["numpy_torch_max_abs_error"],"test_model_sha256":hashlib.sha256((ROOT/"models/visibility_prior.npz").read_bytes()).hexdigest()})
for prefix,report in [("rendering",a),("followup",b)]:
rows=list(csv.DictReader((ROOT/f"results/{prefix}_raw.csv").open()))
finite=all(np.isfinite(float(r[k])) for r in rows for k in ["mse","conditional_expected_mse","mean_error","negative_channel_fraction"])
total=sum(int(r["rays"]) for r in rows)
ray_budget=np.prod(report["protocol"]["viewport"])*report["protocol"]["rays_per_receiver_per_frame"]
ray_match=all(int(r["rays"])==ray_budget for r in rows)
errors=[]
for item in report["summary"]:
raw=[float(r["conditional_expected_mse"]) for r in rows if r["phase"]==item["phase"] and r["method"]==item["method"]]
errors.append(abs(np.mean(raw)-item["expected_mse"]))
gate(f"raw_data_consistency_{prefix}",finite and ray_match and len(rows)==report["record_count"] and total==report["total_physical_ray_calls"] and max(errors)<1e-12,
{"rows":len(rows),"rays":total,"equal_per_row_ray_budget":int(ray_budget),"max_summary_error":float(max(errors))})
original_failure=next(c for c in a["comparisons"] if c["phase"]=="hidden_change" and c["contender"]=="world_active_cv" and c["baseline"]=="world_cv")
neutral_prior=next(c for c in a["comparisons"] if c["phase"]=="revisit" and c["baseline"]=="constant_world_cv")
gate("negative_and_inconclusive_results_retained",original_failure["relative_expected_mse_reduction"]<0 and neutral_prior["scene_bootstrap_95"][0]<0<neutral_prior["scene_bootstrap_95"][1],
{"active_hidden_change_reduction":original_failure["relative_expected_mse_reduction"],"neural_ablation_interval":neutral_prior["scene_bootstrap_95"]})
if require_replay:
errors=[]
for prefix,reference in [("rendering",a),("followup",b)]:
replay=load(f"reproduced_results/{prefix}_report.json")
for original,rerun in zip(reference["summary"],replay["summary"],strict=True):
for key in ("expected_mse","observed_mse","negative_channel_fraction"):
errors.append(abs(original[key]-rerun[key]))
gate("deterministic_full_replay",max(errors)<1e-12,{"max_nontiming_metric_difference":float(max(errors)),"timings_excluded":True})
else:
gate("recorded_replay_evidence",(ROOT/"results/replay_validation.json").exists(),{"live_replay":"Use --require-replay after running both scripts with default output paths"})
install=load("results/installation_validation.json")
gate("recorded_installation_check",install.get("installation_import_passed") is True,{"version":install.get("version"),"meaning":"Recorded clean-target install/import; not a platform compatibility matrix"})
demo=load("results/demo/demo.json")
gate("recorded_physical_demo",np.isfinite(demo["final_mse"]) and demo["frames"]==24 and (ROOT/"results/demo/demo.png").exists(),
{"frames":demo["frames"],"rays":demo["rays"],"mse":demo["final_mse"]})
status=load("STATUS.json")
gate("scope_and_claims_agree",status["verified"]["gpu_benchmarks"]==0 and status["verified"]["full_sr_rr_fg_experiments"]==0 and status["verified"]["new_contract_tests"]==counts[0] and status["overall_maturity_estimate_percent"]<100,
{"production_gates_open":status["production_gates_open"],"research_maturity_is_estimated_not_measured":True})
report={"version":"2.0.0","scope":"Executable finite-light CPU research reference. This checklist does not measure general research maturity or production readiness.",
"passed":all(g["passed"] for g in gates),"gates_passed":sum(g["passed"] for g in gates),"gates_total":len(gates),
"gates":gates,"environment":{"python":platform.python_version(),"numpy":np.__version__},
"not_validated":["Windows launcher execution","Hugging Face upload execution","CUDA or GPU performance","Full SR/RR/FG integration","Independent replication"]}
path=ROOT/output;path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(report,indent=2)+"\n")
print(json.dumps(report,indent=2))
return 0 if report["passed"] else 1
if __name__=="__main__":
parser=argparse.ArgumentParser();parser.add_argument("--output",default="validation_reproduced.json")
parser.add_argument("--require-replay",action="store_true")
args=parser.parse_args();raise SystemExit(validate(args.output,args.require_replay))
|