| |
| """FINAL confirmation run on a FRESH held-out set (seeds 5000-5049), after the two post-hoc fixes (compound-aware |
| handled facts; expiring waits). Nothing here was looked at before this run. Writes results/fresh_eval.json.""" |
| import copy, json, os, sys |
| HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE) |
| sys.path.insert(0, ROOT); sys.path.insert(0, HERE) |
| os.environ["COMPOUND_START"] = "6000" |
| import eval_paper as E |
| from deepspace.fdir import FDIR |
| from deepspace.runtime import run, scenario |
| from deepspace.selfmaint import Rulebook, propose |
| from deepspace.sim import PROFILES |
| FRESH = range(5000, 5050) |
| out = {"seeds": {"fresh_single": "5000-5049", "fresh_compound": "6000-6019"}} |
| print("== FRESH single faults (1,700 scenarios)") |
| out["deepspace"] = E.evaluate(lambda s: FDIR(), "oracle7-deepspace", FRESH) |
| out["expert"] = E.evaluate(lambda s: E.ExpertTable(), "expert fault table", FRESH) |
| out["threshold"] = E.evaluate(lambda s: E.Threshold(), "limit -> safe mode", FRESH) |
| out["random"] = E.evaluate(lambda s: E.Random(s), "random admissible", FRESH) |
| out["noaction"] = E.evaluate(lambda s: E.NoAction(), "no action", FRESH) |
| traces = [run(scenario(f, s, p), FDIR().decide) for p, pr in PROFILES.items() for f in pr["faults"] for s in range(8)] |
| rb = Rulebook() |
| for r in propose(traces): |
| rb.consider(r) |
| out["learned_rules"] = len(rb.current) |
| out["deepspace_learned"] = E.evaluate(lambda s: FDIR(rulebook=copy.deepcopy(rb.current)), "deepspace + learned rules", FRESH) |
| print("== FRESH compound faults (240)") |
| out["compound_deepspace"] = E.compound(lambda s: FDIR(), "oracle7-deepspace") |
| out["compound_expert"] = E.compound(lambda s: E.ExpertTable(), "expert fault table") |
| out["compound_threshold"] = E.compound(lambda s: E.Threshold(), "limit -> safe mode") |
| json.dump(out, open(os.path.join(ROOT, "results", "fresh_eval.json"), "w"), indent=1, default=str) |
| print("written results/fresh_eval.json") |
|
|
