File size: 5,868 Bytes
3538060 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | #!/usr/bin/env python3
"""STRUCTURAL MISMATCH and OUT-OF-RANGE faults. Written, then run ONCE on seeds 7000-7049 (never used before), with
the flight code frozen (SHA-256 recorded; no change to deepspace/ is allowed after seeing these results).
A structural mismatch: the spacecraft flies TRUTH physics (eval/truth.py) the onboard twin does not model -
thermal (conduction, T^4 loss, heater power x0.8-1.25), power (coulombic loss, drifting unmodelled load, noise),
sensors (quantisation, heavy-tailed outliers), attitude (3x jitter), and all of them plus 2% telemetry dropout.
B out-of-range intensities: six fault classes injected WEAK (just above the detection thresholds) and STRONG (far
beyond the design range), on base physics and on ALL-terms physics.
Scoring is unchanged (runtime.score). Because a designed response is not necessarily right outside the design range, each
run also records its OUTCOME (for every run, and for misses): good if the craft came to no harm (no damage, no harmful act, battery above 30%,
no zone out of band for more than 10 minutes). The outcome columns were added after the first run; the runs are
deterministic and the strict numbers reproduce exactly. Writes results/structural_eval.json.
"""
import json
import os
import sys
import time
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
sys.path.insert(0, ROOT)
sys.path.insert(0, HERE)
import eval_paper as E # noqa: E402
from eval_final import code_sha256 # noqa: E402
from deepspace.fdir import FDIR # noqa: E402
from deepspace.runtime import PHYSICAL, scenario, score # noqa: E402
from deepspace.sim import PROFILES # noqa: E402
from truth import INTENSITY, SCALABLE, VARIANTS, run_truth # noqa: E402
SEEDS = range(7000, 7050)
SYSTEMS = {"oracle7-deepspace": lambda s: FDIR(), "expert fault table": lambda s: E.ExpertTable(),
"limit -> safe mode (simplified)": lambda s: E.Threshold()}
def fly(make, terms, faults=None, override=None):
k = harm = dmg = safe = n = trap_n = trap_bad = 0
benign = bad_miss = bad_outcome = 0
socs, sci = [], 0
per = {}
for prof, pr in PROFILES.items():
for fault in pr["faults"]:
if faults and fault not in faults:
continue
c = 0
for seed in SEEDS:
p = scenario(fault, seed, prof)
tr = run_truth(p, make(seed).decide, terms, physical=PHYSICAL, magnitude_override=override)
s = score(p, tr)
k += s["ok"]; c += s["ok"]; harm += s["harmful"]; dmg += s["damage"]; safe += s["safe_mode"]; n += 1
socs.append(tr["min_soc"]); sci += tr["science_off_ticks"]
# OUTCOME of every run, whatever was chosen: no damage, no harmful act, battery above 30%, no zone out of
# band for more than 10 minutes
good = not s["damage"] and not s["harmful"] and tr["min_soc"] > 0.3 and tr["out_of_band_ticks"] <= 10
bad_outcome += not good
if not s["ok"]:
# a miss is BENIGN when the craft came to no harm: no damage, no harmful act, battery above 30%,
# and no zone out of its operating band for more than 10 minutes
benign += good
bad_miss += not good
if fault in ("nominal", "seu_spike", "sensor_bias"):
trap_n += 1
trap_bad += bool(tr["actions"]) and not s["ok"]
per["%s/%s" % (prof, fault)] = [c, len(SEEDS)]
return {"correct": E.wilson(k, n), "correct_n": k, "n": n, "harmful_n": harm, "damage_n": dmg, "safe_mode_n": safe,
"trap_wrong": E.wilson(trap_bad, trap_n) if trap_n else None, "per": per,
"misses_benign": benign, "misses_with_consequence": bad_miss,
"outcome_ok": E.wilson(n - bad_outcome, n), "bad_outcomes": bad_outcome, "min_soc_p5": round(sorted(socs)[len(socs) // 20], 3),
"science_off_ticks_mean": round(sci / n, 1)}
def line(label, r):
print(" %-44s correct %4d/%d = %.3f harmful %3d damage %3d safe %4d | misses benign %3d consequential %3d outcome-ok %.3f soc-p5 %.2f sci-off %.0f" % (
label, r["correct_n"], r["n"], r["correct"][0], r["harmful_n"], r["damage_n"], r["safe_mode_n"], r["misses_benign"],
r["misses_with_consequence"], r["outcome_ok"][0], r["min_soc_p5"], r["science_off_ticks_mean"]), flush=True)
def main():
t0 = time.time()
out = {"code_sha256": code_sha256(), "seeds": "7000-7049", "structural": {}, "intensity": {}}
print("code", out["code_sha256"])
print("== A. STRUCTURAL MISMATCH (truth physics the twin does not model)")
for v, terms in VARIANTS.items():
systems = SYSTEMS if v in ("base", "all") else {"oracle7-deepspace": SYSTEMS["oracle7-deepspace"]}
out["structural"][v] = {}
for name, make in systems.items():
r = fly(make, terms)
out["structural"][v][name] = r
line("%s | %s" % (v, name), r)
print("== B. OUT-OF-RANGE FAULT INTENSITIES")
for phys in ("base", "all"):
for kind, fn in INTENSITY.items():
key = "%s/%s" % (phys, kind)
out["intensity"][key] = {}
for name, make in SYSTEMS.items():
r = fly(make, VARIANTS[phys], faults=SCALABLE, override=fn)
out["intensity"][key][name] = r
line("%s | %s" % (key, name), r)
out["wall_seconds"] = round(time.time() - t0)
json.dump(out, open(os.path.join(ROOT, "results", "structural_eval.json"), "w"), indent=1, default=str)
print("written results/structural_eval.json in %d s" % out["wall_seconds"])
if __name__ == "__main__":
main()
|