eval/eval_structural.py

#!/usr/bin/env python3
"""STRUCTURAL MISMATCH and OUT-OF-RANGE faults. Written, then run ONCE on seeds 7000-7049 (never used before), with
the flight code frozen (SHA-256 recorded; no change to deepspace/ is allowed after seeing these results).

A  structural mismatch: the spacecraft flies TRUTH physics (eval/truth.py) the onboard twin does not model -
   thermal (conduction, T^4 loss, heater power x0.8-1.25), power (coulombic loss, drifting unmodelled load, noise),
   sensors (quantisation, heavy-tailed outliers), attitude (3x jitter), and all of them plus 2% telemetry dropout.
B  out-of-range intensities: six fault classes injected WEAK (just above the detection thresholds) and STRONG (far
   beyond the design range), on base physics and on ALL-terms physics.
Scoring is unchanged (runtime.score). Because a designed response is not necessarily right outside the design range, each
run also records its OUTCOME (for every run, and for misses): good if the craft came to no harm (no damage, no harmful act, battery above 30%,
no zone out of band for more than 10 minutes). The outcome columns were added after the first run; the runs are
deterministic and the strict numbers reproduce exactly. Writes results/structural_eval.json.
"""
import json
import os
import sys
import time

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
sys.path.insert(0, ROOT)
sys.path.insert(0, HERE)
import eval_paper as E  # noqa: E402
from eval_final import code_sha256  # noqa: E402
from deepspace.fdir import FDIR  # noqa: E402
from deepspace.runtime import PHYSICAL, scenario, score  # noqa: E402
from deepspace.sim import PROFILES  # noqa: E402
from truth import INTENSITY, SCALABLE, VARIANTS, run_truth  # noqa: E402

SEEDS = range(7000, 7050)
SYSTEMS = {"oracle7-deepspace": lambda s: FDIR(), "expert fault table": lambda s: E.ExpertTable(),
           "limit -> safe mode (simplified)": lambda s: E.Threshold()}


def fly(make, terms, faults=None, override=None):
    k = harm = dmg = safe = n = trap_n = trap_bad = 0
    benign = bad_miss = bad_outcome = 0
    socs, sci = [], 0
    per = {}
    for prof, pr in PROFILES.items():
        for fault in pr["faults"]:
            if faults and fault not in faults:
                continue
            c = 0
            for seed in SEEDS:
                p = scenario(fault, seed, prof)
                tr = run_truth(p, make(seed).decide, terms, physical=PHYSICAL, magnitude_override=override)
                s = score(p, tr)
                k += s["ok"]; c += s["ok"]; harm += s["harmful"]; dmg += s["damage"]; safe += s["safe_mode"]; n += 1
                socs.append(tr["min_soc"]); sci += tr["science_off_ticks"]
                # OUTCOME of every run, whatever was chosen: no damage, no harmful act, battery above 30%, no zone out of
                # band for more than 10 minutes
                good = not s["damage"] and not s["harmful"] and tr["min_soc"] > 0.3 and tr["out_of_band_ticks"] <= 10
                bad_outcome += not good
                if not s["ok"]:
                    # a miss is BENIGN when the craft came to no harm: no damage, no harmful act, battery above 30%,
                    # and no zone out of its operating band for more than 10 minutes
                    benign += good
                    bad_miss += not good
                if fault in ("nominal", "seu_spike", "sensor_bias"):
                    trap_n += 1
                    trap_bad += bool(tr["actions"]) and not s["ok"]
            per["%s/%s" % (prof, fault)] = [c, len(SEEDS)]
    return {"correct": E.wilson(k, n), "correct_n": k, "n": n, "harmful_n": harm, "damage_n": dmg, "safe_mode_n": safe,
            "trap_wrong": E.wilson(trap_bad, trap_n) if trap_n else None, "per": per,
            "misses_benign": benign, "misses_with_consequence": bad_miss,
            "outcome_ok": E.wilson(n - bad_outcome, n), "bad_outcomes": bad_outcome, "min_soc_p5": round(sorted(socs)[len(socs) // 20], 3),
            "science_off_ticks_mean": round(sci / n, 1)}


def line(label, r):
    print("  %-44s correct %4d/%d = %.3f  harmful %3d  damage %3d  safe %4d | misses benign %3d consequential %3d  outcome-ok %.3f  soc-p5 %.2f  sci-off %.0f" % (
        label, r["correct_n"], r["n"], r["correct"][0], r["harmful_n"], r["damage_n"], r["safe_mode_n"], r["misses_benign"],
        r["misses_with_consequence"], r["outcome_ok"][0], r["min_soc_p5"], r["science_off_ticks_mean"]), flush=True)


def main():
    t0 = time.time()
    out = {"code_sha256": code_sha256(), "seeds": "7000-7049", "structural": {}, "intensity": {}}
    print("code", out["code_sha256"])
    print("== A. STRUCTURAL MISMATCH (truth physics the twin does not model)")
    for v, terms in VARIANTS.items():
        systems = SYSTEMS if v in ("base", "all") else {"oracle7-deepspace": SYSTEMS["oracle7-deepspace"]}
        out["structural"][v] = {}
        for name, make in systems.items():
            r = fly(make, terms)
            out["structural"][v][name] = r
            line("%s | %s" % (v, name), r)
    print("== B. OUT-OF-RANGE FAULT INTENSITIES")
    for phys in ("base", "all"):
        for kind, fn in INTENSITY.items():
            key = "%s/%s" % (phys, kind)
            out["intensity"][key] = {}
            for name, make in SYSTEMS.items():
                r = fly(make, VARIANTS[phys], faults=SCALABLE, override=fn)
                out["intensity"][key][name] = r
                line("%s | %s" % (key, name), r)
    out["wall_seconds"] = round(time.time() - t0)
    json.dump(out, open(os.path.join(ROOT, "results", "structural_eval.json"), "w"), indent=1, default=str)
    print("written results/structural_eval.json in %d s" % out["wall_seconds"])


if __name__ == "__main__":
    main()