Snapshot of covenant main (full git history on GitHub)
Browse files- .gitignore +11 -0
- covenant_failover.py +37 -1
- covenant_highway.py +470 -0
- covenant_onboard.py +738 -0
- covenant_one.py +18 -0
.gitignore
CHANGED
|
@@ -274,6 +274,8 @@ ops/distill_rejected_live.jsonl
|
|
| 274 |
ops/heal.jsonl
|
| 275 |
ops/immune.jsonl
|
| 276 |
ops/own_work_state.json
|
|
|
|
|
|
|
| 277 |
# grid guard runtime state (config files grid_signers/interlocks/policy ARE tracked)
|
| 278 |
ops/grid_commands.jsonl
|
| 279 |
ops/grid_commands.chain.jsonl
|
|
@@ -285,6 +287,13 @@ ops/pause/
|
|
| 285 |
# Node A's off-box peers (the phone's tailnet address), read by covenant_watchdog.py
|
| 286 |
# and covenant_prod.bat alike, so neither public file carries the address.
|
| 287 |
ops/local_peers.txt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
# the laptop site auto-deploy job's per-pass record (tools/site_autodeploy.py), machine-local
|
| 289 |
ops/site_autodeploy.jsonl
|
| 290 |
ops/local_unison_feed.txt
|
|
@@ -435,6 +444,8 @@ ops/model_asks.jsonl
|
|
| 435 |
ops/model_asks.jsonl.prev
|
| 436 |
# A294 (2026-10-07): the threefold witness's readings and its first-seen gaps
|
| 437 |
ops/threefold_witness.json
|
|
|
|
|
|
|
| 438 |
# A375 (2026-10-27): the spiking witness's negatives-accrual pipeline -- runtime state, accrues over months
|
| 439 |
ops/witness_state.json
|
| 440 |
ops/witness_negatives.jsonl
|
|
|
|
| 274 |
ops/heal.jsonl
|
| 275 |
ops/immune.jsonl
|
| 276 |
ops/own_work_state.json
|
| 277 |
+
# laptop auto-push pass log (tools/repo_autopush.py): a local operational record, not corpus state
|
| 278 |
+
ops/repo_autopush.jsonl
|
| 279 |
# grid guard runtime state (config files grid_signers/interlocks/policy ARE tracked)
|
| 280 |
ops/grid_commands.jsonl
|
| 281 |
ops/grid_commands.chain.jsonl
|
|
|
|
| 287 |
# Node A's off-box peers (the phone's tailnet address), read by covenant_watchdog.py
|
| 288 |
# and covenant_prod.bat alike, so neither public file carries the address.
|
| 289 |
ops/local_peers.txt
|
| 290 |
+
# the AUTO_GATED auto-onboard lane's LOCAL state (covenant_onboard.py): the operator's
|
| 291 |
+
# arm/trust-root switch and the tasks the lane has armed. Both machine-local decisions,
|
| 292 |
+
# not repo content -- the lane ships DARK (no arm file) and writes armed_tasks only once
|
| 293 |
+
# armed. NODE_ROSTER.json and node_refusals.json ARE tracked (the whois trust root needs
|
| 294 |
+
# the roster committed; a refusal only tightens).
|
| 295 |
+
ops/AUTO_ONBOARD_ARMED
|
| 296 |
+
ops/armed_tasks.json
|
| 297 |
# the laptop site auto-deploy job's per-pass record (tools/site_autodeploy.py), machine-local
|
| 298 |
ops/site_autodeploy.jsonl
|
| 299 |
ops/local_unison_feed.txt
|
|
|
|
| 444 |
ops/model_asks.jsonl.prev
|
| 445 |
# A294 (2026-10-07): the threefold witness's readings and its first-seen gaps
|
| 446 |
ops/threefold_witness.json
|
| 447 |
+
# A405 (2026-11-02): the highway's per-copy mirror-lag cycle counts (first-seen lag, cycles behind the unison target)
|
| 448 |
+
ops/unison_lag_witness.json
|
| 449 |
# A375 (2026-10-27): the spiking witness's negatives-accrual pipeline -- runtime state, accrues over months
|
| 450 |
ops/witness_state.json
|
| 451 |
ops/witness_negatives.jsonl
|
covenant_failover.py
CHANGED
|
@@ -84,6 +84,7 @@ SPOF_REMIND_MARKER = os.path.join(HERE, "ops", "REDUNDANCY_SPOF_REMIND")
|
|
| 84 |
BEACON = os.path.join(HERE, "covenant_beacon.py")
|
| 85 |
BEACON_WIN_PORT = 8771
|
| 86 |
BEACON_WSL_PORT = 8772
|
|
|
|
| 87 |
GITHOST_STANDBY = os.path.join(HERE, "covenant_githost_standby.py")
|
| 88 |
GITHOST_STANDBY_PORT = 8089
|
| 89 |
|
|
@@ -188,6 +189,9 @@ def _probe_instance(inst, probe=None):
|
|
| 188 |
if p.get("type") == "feed":
|
| 189 |
url = _feed_url()
|
| 190 |
return _http_probe(url) if url else None # no private feed URL configured here -> UNKNOWN
|
|
|
|
|
|
|
|
|
|
| 191 |
return None # cannot probe here -> UNKNOWN for this instance
|
| 192 |
|
| 193 |
|
|
@@ -205,6 +209,22 @@ def _feed_url():
|
|
| 205 |
return None
|
| 206 |
|
| 207 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
def _inst_id(inst):
|
| 209 |
return "%s/%s@%s" % (inst.get("name") or "?", inst.get("env") or "?", inst.get("host") or "?")
|
| 210 |
|
|
@@ -278,7 +298,7 @@ SERVICES = {
|
|
| 278 |
# The one service this module actually provisions as a live cross-OS pair.
|
| 279 |
# DARK until ops/REDUNDANCY_ARMED exists (RAM budget).
|
| 280 |
"beacon": {
|
| 281 |
-
"desc": "the redundancy read-API itself -- a provisioned cross-OS pair",
|
| 282 |
"require_redundant": True, "min_environments": 2, "redundancy_model": "hot",
|
| 283 |
"failover_class": AUTO_REVERSIBLE, "arm_gated": True,
|
| 284 |
"instances": [
|
|
@@ -288,6 +308,19 @@ SERVICES = {
|
|
| 288 |
{"name": "beacon-wsl", "kind": "redundant", "env": "wsl-ubuntu", "host": "laptop",
|
| 289 |
"provisioned": True, "hot": True, "ram_mb": 15, "launch": "wsl-ubuntu", "port": BEACON_WSL_PORT,
|
| 290 |
"probe": {"type": "http", "url": "http://127.0.0.1:%d/beacon" % BEACON_WSL_PORT}},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
],
|
| 292 |
},
|
| 293 |
}
|
|
@@ -530,6 +563,9 @@ def _start_instance(inst, dry_run):
|
|
| 530 |
return _launch_windows_beacon(inst, dry_run)
|
| 531 |
if launch == "wsl-ubuntu":
|
| 532 |
return _launch_wsl_beacon(inst, dry_run)
|
|
|
|
|
|
|
|
|
|
| 533 |
return False, "no known launcher for %s (provision it first -- PROPOSE_ONLY)" % inst.get("name")
|
| 534 |
|
| 535 |
|
|
|
|
| 84 |
BEACON = os.path.join(HERE, "covenant_beacon.py")
|
| 85 |
BEACON_WIN_PORT = 8771
|
| 86 |
BEACON_WSL_PORT = 8772
|
| 87 |
+
BEACON_PHONE_PORT = 8773
|
| 88 |
GITHOST_STANDBY = os.path.join(HERE, "covenant_githost_standby.py")
|
| 89 |
GITHOST_STANDBY_PORT = 8089
|
| 90 |
|
|
|
|
| 189 |
if p.get("type") == "feed":
|
| 190 |
url = _feed_url()
|
| 191 |
return _http_probe(url) if url else None # no private feed URL configured here -> UNKNOWN
|
| 192 |
+
if p.get("type") == "phone":
|
| 193 |
+
url = _phone_beacon_url()
|
| 194 |
+
return _http_probe(url) if url else None # no phone address configured here -> UNKNOWN (rule 9)
|
| 195 |
return None # cannot probe here -> UNKNOWN for this instance
|
| 196 |
|
| 197 |
|
|
|
|
| 209 |
return None
|
| 210 |
|
| 211 |
|
| 212 |
+
def _phone_beacon_url():
|
| 213 |
+
"""The phone beacon's real URL -- env COVENANT_PHONE_BEACON, else the first line of the
|
| 214 |
+
gitignored ops/local_phone_beacon.txt (the phone's tailnet address lives in a private file,
|
| 215 |
+
never in this public repo, exactly like the unison feed). None where neither is set, so an
|
| 216 |
+
intermittent phone that is asleep -- or has never been provisioned -- reads UNKNOWN here,
|
| 217 |
+
never 'down' (rule 9): its absence must not masquerade as a failure OR as redundancy."""
|
| 218 |
+
url = os.environ.get("COVENANT_PHONE_BEACON", "").strip()
|
| 219 |
+
if url:
|
| 220 |
+
return url
|
| 221 |
+
try:
|
| 222 |
+
with open(os.path.join(HERE, "ops", "local_phone_beacon.txt"), encoding="utf-8") as fh:
|
| 223 |
+
return fh.readline().strip() or None
|
| 224 |
+
except OSError:
|
| 225 |
+
return None
|
| 226 |
+
|
| 227 |
+
|
| 228 |
def _inst_id(inst):
|
| 229 |
return "%s/%s@%s" % (inst.get("name") or "?", inst.get("env") or "?", inst.get("host") or "?")
|
| 230 |
|
|
|
|
| 298 |
# The one service this module actually provisions as a live cross-OS pair.
|
| 299 |
# DARK until ops/REDUNDANCY_ARMED exists (RAM budget).
|
| 300 |
"beacon": {
|
| 301 |
+
"desc": "the redundancy read-API itself -- a provisioned cross-OS pair (+ an intermittent phone third env)",
|
| 302 |
"require_redundant": True, "min_environments": 2, "redundancy_model": "hot",
|
| 303 |
"failover_class": AUTO_REVERSIBLE, "arm_gated": True,
|
| 304 |
"instances": [
|
|
|
|
| 308 |
{"name": "beacon-wsl", "kind": "redundant", "env": "wsl-ubuntu", "host": "laptop",
|
| 309 |
"provisioned": True, "hot": True, "ram_mb": 15, "launch": "wsl-ubuntu", "port": BEACON_WSL_PORT,
|
| 310 |
"probe": {"type": "http", "url": "http://127.0.0.1:%d/beacon" % BEACON_WSL_PORT}},
|
| 311 |
+
# A THIRD environment, on the phone's own device (Galaxy S25+) -- survives the loss of BOTH
|
| 312 |
+
# laptop OSes (the operator, 2026-11-03: "make redundancies on the phone too"). Witness-class:
|
| 313 |
+
# the phone is INTERMITTENT (it sleeps) and is provisioned ONLY by an APP INSTALL on the phone,
|
| 314 |
+
# never started from the laptop (nothing pushes to the phone -- launch "phone" is declined).
|
| 315 |
+
# provisioned=False until the phone node actually runs the beacon AND its tailnet URL is in
|
| 316 |
+
# COVENANT_PHONE_BEACON / ops/local_phone_beacon.txt; until then it is DECLARED, not counted
|
| 317 |
+
# (so it is neither a false SPOF nor a phantom standby). It rides on the headscale join
|
| 318 |
+
# (phone-app-join). min_environments stays 2 -- the two laptop OSes carry "healthy"; the phone
|
| 319 |
+
# is a bonus third env when awake, and when it is provisioned+awake a laptop env dying while the
|
| 320 |
+
# phone is asleep correctly reads degraded.
|
| 321 |
+
{"name": "beacon-phone", "kind": "redundant", "env": "phone", "host": "phone",
|
| 322 |
+
"provisioned": False, "hot": True, "ram_mb": 15, "intermittent": True,
|
| 323 |
+
"launch": "phone", "port": BEACON_PHONE_PORT, "probe": {"type": "phone"}},
|
| 324 |
],
|
| 325 |
},
|
| 326 |
}
|
|
|
|
| 563 |
return _launch_windows_beacon(inst, dry_run)
|
| 564 |
if launch == "wsl-ubuntu":
|
| 565 |
return _launch_wsl_beacon(inst, dry_run)
|
| 566 |
+
if launch == "phone":
|
| 567 |
+
return False, ("the phone beacon is provisioned by an APP INSTALL on the phone and never started "
|
| 568 |
+
"from the laptop (nothing pushes to the phone) -- coordinate with the phone node")
|
| 569 |
return False, "no known launcher for %s (provision it first -- PROPOSE_ONLY)" % inst.get("name")
|
| 570 |
|
| 571 |
|
covenant_highway.py
CHANGED
|
@@ -2378,6 +2378,373 @@ def detect_git_host_down(health=None, url=None, marker=None, **kwargs):
|
|
| 2378 |
"url": url, "error": "%s: %s" % (type(e).__name__, str(e)[:120])}}
|
| 2379 |
|
| 2380 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2381 |
DETECTORS = {
|
| 2382 |
"node_down": detect_node_down,
|
| 2383 |
"tetsu_cannot_answer": detect_tetsu_cannot_answer,
|
|
@@ -2401,6 +2768,14 @@ DETECTORS = {
|
|
| 2401 |
"sweep_not_current": detect_sweep_not_current,
|
| 2402 |
"ambassador_stalled": detect_ambassador_stalled,
|
| 2403 |
"schedule_stalled": detect_schedule_stalled,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2404 |
"moltbook_strikes": detect_moltbook_strikes,
|
| 2405 |
"threefold_witness": detect_threefold_witness,
|
| 2406 |
# "public_email_exposure" RETIRED the day it was added (A306), his words 2026-10-07: "they aren't issues idgaf".
|
|
@@ -3051,7 +3426,86 @@ def remedy_refresh_defender(measured, dry_run=True):
|
|
| 3051 |
scanned, "; ".join(mendable))
|
| 3052 |
|
| 3053 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3054 |
REMEDIES = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3055 |
"refresh_defender": {"fn": remedy_refresh_defender, "klass": AUTO_REVERSIBLE,
|
| 3056 |
"for": ["defense_lapse"], "kind": "stateless",
|
| 3057 |
"touches": ["Defender's signature update, judged by covenant_immune.update(), and a scan of our own tree; never its settings"],
|
|
@@ -4084,6 +4538,22 @@ def run_once(dry_run=False, exclude=("restart_watchdog",), ledger=None, health=N
|
|
| 4084 |
# repairs it" without that is a nudge, not a finding.
|
| 4085 |
alerts.append("highway: %s is present and nothing here repairs it -- %s"
|
| 4086 |
% (name, json.dumps(c["measured"])[:400]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4087 |
return alerts, infos
|
| 4088 |
|
| 4089 |
|
|
|
|
| 2378 |
"url": url, "error": "%s: %s" % (type(e).__name__, str(e)[:120])}}
|
| 2379 |
|
| 2380 |
|
| 2381 |
+
# ----------------------------------------------------------------- the sync fabric (A405, 2026-11-02)
|
| 2382 |
+
#
|
| 2383 |
+
# His words, 2026-11-02: "continue growing the mycelial highway. Make it pave itself." The highway already
|
| 2384 |
+
# self-paves what it has always owned; today a new SYNC FABRIC was built around it and nothing in this file
|
| 2385 |
+
# could see it. The fabric is: a UNISON FEED served on TETSU (:8800/unison), one reading every `interval_s`,
|
| 2386 |
+
# that says for each repo whether its github / hf / website copies have reached a common `target`; and two
|
| 2387 |
+
# operator-armed scheduled tasks that act on that feed -- CovenantHFMirror (repo history -> Hugging Face,
|
| 2388 |
+
# fast-forward-only, opsec-gated, visibility-preserving) and CovenantSiteAutoDeploy (GitHub -> the website's
|
| 2389 |
+
# repo mirror, opsec-gated, never a half-sync). A GitHub pusher, CovenantRepoAutopush (open-PR / opsec / ff
|
| 2390 |
+
# gated), is being built in a sibling session and may not be armed yet.
|
| 2391 |
+
#
|
| 2392 |
+
# These detectors let the loop SENSE that fabric, and -- only in the one narrow case that is strictly
|
| 2393 |
+
# reversible: re-running an already-armed, guarded, idempotent task -- reversibly HEAL it. The line the
|
| 2394 |
+
# operator is holding does NOT move here. Nothing auto-ONBOARDS, auto-INSTALLS or auto-ARMS a node or a task
|
| 2395 |
+
# (arming a task is config, PROPOSE_ONLY by construction). A public GitHub push is NEVER AUTO_REVERSIBLE here;
|
| 2396 |
+
# it belongs to repo_autopush, itself gated. Keys, private holdings and the portfolio never ride the fabric
|
| 2397 |
+
# and are not read by anything below.
|
| 2398 |
+
#
|
| 2399 |
+
# SAFE WHEN THE LOOP RUNS DRY (A244, highway_may_act). The detectors only READ (a feed GET, a scheduler read);
|
| 2400 |
+
# reading is safe in a staged/worktree copy. The remedies execute nothing when dry_run is True (the
|
| 2401 |
+
# remedy_restart_nodes pattern), so a staged pass that senses the live fabric still touches nothing.
|
| 2402 |
+
#
|
| 2403 |
+
# WHERE THE FEED'S ADDRESS LIVES. His network addresses never go in this public repo (the rule ops/local_peers.txt
|
| 2404 |
+
# and tools/site_autodeploy.py already follow). The feed URL is read from COVENANT_UNISON_FEED or the first line
|
| 2405 |
+
# of the gitignored ops/local_unison_feed.txt; with neither set the feed is "not configured here" and the feed
|
| 2406 |
+
# detectors read UNKNOWN -- never OK (rule 9, P20).
|
| 2407 |
+
|
| 2408 |
+
UNISON_FEED_LOCAL = os.path.join(HERE, "ops", "local_unison_feed.txt")
|
| 2409 |
+
UNISON_FEED_CACHE_S = 120 # the three feed detectors in one sense() pass share one GET; the feed moves every interval_s
|
| 2410 |
+
UNISON_COPIES = ("github", "hf", "website")
|
| 2411 |
+
# The tasks that act on the fabric, and the copy each one keeps in step (so a lagging copy is re-synced by the
|
| 2412 |
+
# task that owns it). RepoAutopush owns the github copy and is OPTIONAL -- checked only if it is armed.
|
| 2413 |
+
SYNC_TASKS_EXPECTED = ("CovenantHFMirror", "CovenantSiteAutoDeploy")
|
| 2414 |
+
SYNC_TASKS_OPTIONAL = ("CovenantRepoAutopush",)
|
| 2415 |
+
COPY_TASK = {"hf": "CovenantHFMirror", "website": "CovenantSiteAutoDeploy", "github": "CovenantRepoAutopush"}
|
| 2416 |
+
STALE_RUN_INTERVALS = 3 # a task whose last run is older than this many of its OWN intervals is not firing
|
| 2417 |
+
MIRROR_LAG_CYCLES = 3 # a copy behind the target across more than this many DISTINCT feed readings
|
| 2418 |
+
UNISON_LAG_CACHE = os.path.join(HERE, "ops", "unison_lag_witness.json") # gitignored: first-seen lag and cycle counts
|
| 2419 |
+
# LastTaskResult codes that are NOT a failed run: 0 is success; 0x4130x are Windows task-STATE codes (ready,
|
| 2420 |
+
# running, not-yet-run, terminated, ...) that a scheduler can report in place of an exit code. Treating them as
|
| 2421 |
+
# non-failures keeps a task mid-run or freshly armed from reading as a failure (a false positive that would
|
| 2422 |
+
# re-run a task that is already running).
|
| 2423 |
+
_TASK_OK_RESULTS = frozenset({0, 0x41300, 0x41301, 0x41302, 0x41303, 0x41304, 0x41305, 0x41306})
|
| 2424 |
+
|
| 2425 |
+
_sync_task_cache = {"at": 0.0, "tasks": None}
|
| 2426 |
+
_unison_feed_cache = {"at": 0.0, "feed": None, "note": ""}
|
| 2427 |
+
|
| 2428 |
+
|
| 2429 |
+
def _naive_local_ts(s):
|
| 2430 |
+
"""Epoch for a naive ISO stamp like '2026-11-02T13:38:15', read as local time (the feed writes TETSU-local)."""
|
| 2431 |
+
import datetime
|
| 2432 |
+
return datetime.datetime.strptime(str(s)[:19], "%Y-%m-%dT%H:%M:%S").timestamp()
|
| 2433 |
+
|
| 2434 |
+
|
| 2435 |
+
def _iso8601_dur_s(s):
|
| 2436 |
+
"""Seconds in an ISO-8601 duration's TIME part, e.g. 'PT2H'->7200, 'PT15M'->900, 'PT1H30M'->5400; None when
|
| 2437 |
+
there is no readable time part. Discovered from a task's own trigger so a cadence the operator changes is
|
| 2438 |
+
judged by the new cadence, not a number hardcoded here (rule 2)."""
|
| 2439 |
+
import re as _re
|
| 2440 |
+
if not s or not isinstance(s, str):
|
| 2441 |
+
return None
|
| 2442 |
+
m = _re.match(r"^P(?:\d+W)?(?:\d+D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$", s.strip())
|
| 2443 |
+
if not m or not any(m.groups()):
|
| 2444 |
+
return None
|
| 2445 |
+
h, mi, se = (int(x) if x else 0 for x in m.groups())
|
| 2446 |
+
return (h * 3600 + mi * 60 + se) or None
|
| 2447 |
+
|
| 2448 |
+
|
| 2449 |
+
def _unison_feed_url():
|
| 2450 |
+
"""The feed's URL -- env COVENANT_UNISON_FEED, else the first line of gitignored ops/local_unison_feed.txt,
|
| 2451 |
+
else None. His addresses never sit in this public repo (the ops/local_peers.txt rule)."""
|
| 2452 |
+
url = (os.environ.get("COVENANT_UNISON_FEED") or "").strip()
|
| 2453 |
+
if url:
|
| 2454 |
+
return url
|
| 2455 |
+
try:
|
| 2456 |
+
with io.open(UNISON_FEED_LOCAL, encoding="utf-8") as fh:
|
| 2457 |
+
return fh.readline().strip() or None
|
| 2458 |
+
except OSError:
|
| 2459 |
+
return None
|
| 2460 |
+
|
| 2461 |
+
|
| 2462 |
+
def _unison_feed(url=None, get=None):
|
| 2463 |
+
"""(feed|None, note). `get`, when given, is a callable(url)->dict the suite uses in place of the network.
|
| 2464 |
+
note is '' on a clean read, else WHY it could not be read; the 'no feed URL configured' note is how the
|
| 2465 |
+
caller tells 'unconfigured here' (UNKNOWN) apart from 'configured and down' (a finding)."""
|
| 2466 |
+
if get is not None:
|
| 2467 |
+
try:
|
| 2468 |
+
return get(url), ""
|
| 2469 |
+
except Exception as e: # noqa: BLE001
|
| 2470 |
+
return None, "%s: %s" % (type(e).__name__, str(e)[:80])
|
| 2471 |
+
u = url or _unison_feed_url()
|
| 2472 |
+
if not u:
|
| 2473 |
+
return None, "no feed URL configured here (COVENANT_UNISON_FEED / ops/local_unison_feed.txt)"
|
| 2474 |
+
try:
|
| 2475 |
+
return _get(u, timeout=10), ""
|
| 2476 |
+
except Exception as e: # noqa: BLE001
|
| 2477 |
+
return None, "%s: %s" % (type(e).__name__, str(e)[:80])
|
| 2478 |
+
|
| 2479 |
+
|
| 2480 |
+
def _cached_feed():
|
| 2481 |
+
"""The feed, cached for UNISON_FEED_CACHE_S so the three feed detectors in one pass make one GET, not three."""
|
| 2482 |
+
now = time.time()
|
| 2483 |
+
if _unison_feed_cache["feed"] is None or now - _unison_feed_cache["at"] > UNISON_FEED_CACHE_S:
|
| 2484 |
+
feed, note = _unison_feed()
|
| 2485 |
+
_unison_feed_cache.update(at=now, feed=feed, note=note)
|
| 2486 |
+
return _unison_feed_cache["feed"], _unison_feed_cache["note"]
|
| 2487 |
+
|
| 2488 |
+
|
| 2489 |
+
def _resolve_feed(feed, url, get):
|
| 2490 |
+
"""A detector's feed: an explicit `feed` (the suite), else a `url`/`get` read (the suite), else the cache."""
|
| 2491 |
+
if feed is not None:
|
| 2492 |
+
return feed, ""
|
| 2493 |
+
if url is not None or get is not None:
|
| 2494 |
+
return _unison_feed(url, get)
|
| 2495 |
+
return _cached_feed()
|
| 2496 |
+
|
| 2497 |
+
|
| 2498 |
+
def detect_unison_feed_stale(health=None, feed=None, url=None, get=None, now=None):
|
| 2499 |
+
"""The unison feed -- the one reading the whole sync fabric agrees on -- is unreachable, not OK, or old.
|
| 2500 |
+
|
| 2501 |
+
THE GAP THIS CLOSES (2026-11-02). CovenantHFMirror and CovenantSiteAutoDeploy act ONLY on what the feed says
|
| 2502 |
+
is due; site_autodeploy 'never deploys blind' and does nothing when the feed is unready. So a feed that has
|
| 2503 |
+
stopped answering, or is stuck on an old reading, silently freezes the whole fabric: every downstream task
|
| 2504 |
+
reads 'nothing due' and the road reads quiet. This asks the one question none of the others can -- is the
|
| 2505 |
+
feed itself alive and fresh?
|
| 2506 |
+
|
| 2507 |
+
PRESENT when the feed is unreachable, its `status` is not 'ok', or its own `checked` stamp is older than
|
| 2508 |
+
~2x its `interval_s` (a reading it owed and did not refresh). UNKNOWN -- never OK -- when no feed URL is
|
| 2509 |
+
configured here or the read failed in a way that is the reader's blindness, not the feed's fault.
|
| 2510 |
+
|
| 2511 |
+
NO AUTO REMEDY. The feed runs on TETSU; nothing on this machine can restart it, and reaching across to his
|
| 2512 |
+
machine to restart a service is not reversible from here. The remedy is PROPOSE_ONLY (propose_unison_feed):
|
| 2513 |
+
the condition, the covenant's reading, and the cost beside the gain, for a person to act on."""
|
| 2514 |
+
now = time.time() if now is None else now
|
| 2515 |
+
feed, note = _resolve_feed(feed, url, get)
|
| 2516 |
+
if feed is None:
|
| 2517 |
+
# Unconfigured here is the reader's blindness (UNKNOWN); a configured feed that will not answer is the
|
| 2518 |
+
# finding (PRESENT). The two are told apart by whether a URL was set.
|
| 2519 |
+
if "no feed URL configured" in note:
|
| 2520 |
+
return {"state": UNKNOWN, "measured": {"why": note}}
|
| 2521 |
+
return {"state": PRESENT, "measured": {"reachable": False, "why": note or "unreachable"}}
|
| 2522 |
+
if not isinstance(feed, dict):
|
| 2523 |
+
return {"state": PRESENT, "measured": {"reachable": True, "why": "feed is not a JSON object"}}
|
| 2524 |
+
status, interval, checked = feed.get("status"), feed.get("interval_s"), feed.get("checked")
|
| 2525 |
+
age_s = None
|
| 2526 |
+
try:
|
| 2527 |
+
if checked:
|
| 2528 |
+
age_s = now - _naive_local_ts(checked)
|
| 2529 |
+
except Exception: # noqa: BLE001
|
| 2530 |
+
age_s = None
|
| 2531 |
+
stale_after = (2.0 * float(interval)) if isinstance(interval, (int, float)) and interval else None
|
| 2532 |
+
reasons = []
|
| 2533 |
+
if status != "ok":
|
| 2534 |
+
reasons.append("status is %r, not 'ok'" % status)
|
| 2535 |
+
if stale_after is not None and age_s is not None and age_s > stale_after:
|
| 2536 |
+
reasons.append("last checked %.0fs ago, over 2x interval (%.0fs)" % (age_s, stale_after))
|
| 2537 |
+
measured = {"reachable": True, "status": status, "interval_s": interval, "checked": checked,
|
| 2538 |
+
"age_s": None if age_s is None else round(age_s, 1), "reasons": reasons}
|
| 2539 |
+
if stale_after is None:
|
| 2540 |
+
measured["note"] = "no usable interval_s on the feed; staleness not judged, status only"
|
| 2541 |
+
return {"state": PRESENT if reasons else ABSENT, "measured": measured}
|
| 2542 |
+
|
| 2543 |
+
|
| 2544 |
+
def _read_sync_tasks(names, run=None):
|
| 2545 |
+
"""{name: {present, state, last_run(epoch|None), result(int|None), interval_s(|None), next_run(epoch|None)}} for
|
| 2546 |
+
each requested task. None (the whole dict) when the scheduler cannot be read here (not Windows / read failed).
|
| 2547 |
+
`run` is the seam the suite replaces: a callable()->the parsed JSON (list|dict|None), so no PowerShell runs
|
| 2548 |
+
in a test."""
|
| 2549 |
+
if run is None and os.name != "nt":
|
| 2550 |
+
return None
|
| 2551 |
+
if run is not None:
|
| 2552 |
+
data = run()
|
| 2553 |
+
else:
|
| 2554 |
+
quoted = ",".join("'%s'" % n.replace("'", "''") for n in names)
|
| 2555 |
+
ps = ("Get-ScheduledTask -TaskName %s -ErrorAction SilentlyContinue | ForEach-Object { "
|
| 2556 |
+
"$i = $_ | Get-ScheduledTaskInfo; [pscustomobject]@{name=$_.TaskName; state=[string]$_.State; "
|
| 2557 |
+
"last=$(if ($i.LastRunTime -and $i.LastRunTime.Year -gt 2000) {([DateTimeOffset]$i.LastRunTime).ToUnixTimeSeconds()} else {$null}); "
|
| 2558 |
+
"result=$i.LastTaskResult; "
|
| 2559 |
+
"next=$(if ($i.NextRunTime) {([DateTimeOffset]$i.NextRunTime).ToUnixTimeSeconds()} else {$null}); "
|
| 2560 |
+
"interval=[string]($_.Triggers | ForEach-Object { $_.Repetition.Interval } | Where-Object { $_ } | Select-Object -First 1)} } "
|
| 2561 |
+
"| ConvertTo-Json -Compress") % quoted
|
| 2562 |
+
import subprocess
|
| 2563 |
+
try:
|
| 2564 |
+
r = subprocess.run(["powershell", "-NoProfile", "-Command", ps], capture_output=True, text=True,
|
| 2565 |
+
timeout=60, creationflags=_NOWIN)
|
| 2566 |
+
data = json.loads(r.stdout or "null")
|
| 2567 |
+
except Exception: # noqa: BLE001
|
| 2568 |
+
return None
|
| 2569 |
+
data = [] if data is None else (data if isinstance(data, list) else [data])
|
| 2570 |
+
out = {}
|
| 2571 |
+
for d in data:
|
| 2572 |
+
nm = d.get("name")
|
| 2573 |
+
if not nm:
|
| 2574 |
+
continue
|
| 2575 |
+
out[nm] = {"present": True, "state": d.get("state"),
|
| 2576 |
+
"last_run": (float(d["last"]) if d.get("last") is not None else None),
|
| 2577 |
+
"result": (int(d["result"]) if d.get("result") is not None else None),
|
| 2578 |
+
"interval_s": _iso8601_dur_s(d.get("interval")),
|
| 2579 |
+
"next_run": (float(d["next"]) if d.get("next") is not None else None)}
|
| 2580 |
+
for nm in names:
|
| 2581 |
+
out.setdefault(nm, {"present": False})
|
| 2582 |
+
return out
|
| 2583 |
+
|
| 2584 |
+
|
| 2585 |
+
def _cached_sync_tasks():
|
| 2586 |
+
"""The sync tasks' scheduler state, cached for SCHEDULE_EVERY_S (as detect_schedule_stalled caches its read)."""
|
| 2587 |
+
now = time.time()
|
| 2588 |
+
if _sync_task_cache["tasks"] is None or now - _sync_task_cache["at"] > SCHEDULE_EVERY_S:
|
| 2589 |
+
_sync_task_cache.update(at=now, tasks=_read_sync_tasks(SYNC_TASKS_EXPECTED + SYNC_TASKS_OPTIONAL))
|
| 2590 |
+
return _sync_task_cache["tasks"]
|
| 2591 |
+
|
| 2592 |
+
|
| 2593 |
+
def detect_sync_task_not_firing(health=None, tasks=None, now=None):
|
| 2594 |
+
"""An armed sync task that is present and NOT doing its job: its last run FAILED (a nonzero exit that is not a
|
| 2595 |
+
task-state code), or it has not run in ~3x its OWN interval. The one condition the loop may reversibly heal
|
| 2596 |
+
here -- by re-running the same already-armed, guarded, idempotent task (rerun_sync_task, AUTO_REVERSIBLE).
|
| 2597 |
+
|
| 2598 |
+
SEPARATE FROM detect_sync_task_missing, the split detect_build_stale_on_pc was carved out for: a remedy must
|
| 2599 |
+
be graded against the condition it can CLEAR. Re-running a task cannot conjure a task that is not armed;
|
| 2600 |
+
pairing the re-run with 'a task is missing' would grade it 'did not fix' every pass a task stayed unarmed and
|
| 2601 |
+
quarantine the one remedy that heals the tasks that DO exist. Missing-ness is a config gap, the operator's
|
| 2602 |
+
call (PROPOSE_ONLY); detect_sync_task_missing carries it, with no remedy of its own.
|
| 2603 |
+
|
| 2604 |
+
SEPARATE FROM detect_schedule_stalled too: that reads next-run/missed for EVERY Covenant* task and never
|
| 2605 |
+
reads the LAST RESULT, so a task that fires on time and exits nonzero every time reads healthy to it. This
|
| 2606 |
+
reads the exit code, for the sync tasks only, and pairs a remedy. A disabled task is a person's choice and is
|
| 2607 |
+
skipped (its rule). A never-run task with a known interval is not-firing; with no readable interval it is
|
| 2608 |
+
judged on its exit code alone."""
|
| 2609 |
+
now = time.time() if now is None else now
|
| 2610 |
+
tasks = _cached_sync_tasks() if tasks is None else tasks
|
| 2611 |
+
if tasks is None:
|
| 2612 |
+
return {"state": UNKNOWN, "measured": {"why": "the scheduler could not be read here (not Windows, or the read failed)"}}
|
| 2613 |
+
not_firing, why = [], {}
|
| 2614 |
+
for name in SYNC_TASKS_EXPECTED + SYNC_TASKS_OPTIONAL:
|
| 2615 |
+
t = tasks.get(name) or {}
|
| 2616 |
+
if not t.get("present") or str(t.get("state")) == "Disabled":
|
| 2617 |
+
continue
|
| 2618 |
+
res = t.get("result")
|
| 2619 |
+
reason = None
|
| 2620 |
+
if res is not None and res not in _TASK_OK_RESULTS:
|
| 2621 |
+
reason = "last run exited %s (0x%X)" % (res, res & 0xFFFFFFFF)
|
| 2622 |
+
else:
|
| 2623 |
+
interval, last = t.get("interval_s"), t.get("last_run")
|
| 2624 |
+
if interval:
|
| 2625 |
+
if last is None:
|
| 2626 |
+
reason = "has a %.0fs interval but has never run" % interval
|
| 2627 |
+
elif now - last > STALE_RUN_INTERVALS * interval:
|
| 2628 |
+
reason = "last ran %.0fs ago, over %dx its %.0fs interval" % (now - last, STALE_RUN_INTERVALS, interval)
|
| 2629 |
+
if reason:
|
| 2630 |
+
not_firing.append(name)
|
| 2631 |
+
why[name] = reason
|
| 2632 |
+
return {"state": PRESENT if not_firing else ABSENT,
|
| 2633 |
+
"measured": {"checked": list(SYNC_TASKS_EXPECTED + SYNC_TASKS_OPTIONAL), "not_firing": not_firing, "why": why}}
|
| 2634 |
+
|
| 2635 |
+
|
| 2636 |
+
def detect_sync_task_missing(health=None, tasks=None):
|
| 2637 |
+
"""An armed sync task that should be present is gone entirely. ARMING IS CONFIG, so there is NO remedy here:
|
| 2638 |
+
re-arming a task is PROPOSE_ONLY by construction (the operator is holding that decision), and
|
| 2639 |
+
detect_schedule_stalled cannot see an ABSENT task -- it grades only tasks the scheduler still lists. Only the
|
| 2640 |
+
two tasks armed as part of the fabric are expected; CovenantRepoAutopush is optional (it may not be armed yet)
|
| 2641 |
+
and its absence is never flagged."""
|
| 2642 |
+
tasks = _read_sync_tasks(SYNC_TASKS_EXPECTED) if tasks is None else tasks
|
| 2643 |
+
if tasks is None:
|
| 2644 |
+
return {"state": UNKNOWN, "measured": {"why": "the scheduler could not be read here (not Windows, or the read failed)"}}
|
| 2645 |
+
missing = [n for n in SYNC_TASKS_EXPECTED if not (tasks.get(n) or {}).get("present")]
|
| 2646 |
+
return {"state": PRESENT if missing else ABSENT,
|
| 2647 |
+
"measured": {"expected": list(SYNC_TASKS_EXPECTED), "missing": missing,
|
| 2648 |
+
"note": "re-arming a scheduled task is config -- a person's call, never auto (A244 line held)"}}
|
| 2649 |
+
|
| 2650 |
+
|
| 2651 |
+
def _load_lag_cache(path=None):
|
| 2652 |
+
try:
|
| 2653 |
+
with io.open(path or UNISON_LAG_CACHE, encoding="utf-8") as fh:
|
| 2654 |
+
d = json.load(fh)
|
| 2655 |
+
return d if isinstance(d, dict) else {}
|
| 2656 |
+
except (OSError, ValueError):
|
| 2657 |
+
return {}
|
| 2658 |
+
|
| 2659 |
+
|
| 2660 |
+
def _save_lag_cache(d, path=None):
|
| 2661 |
+
p = path or UNISON_LAG_CACHE
|
| 2662 |
+
try:
|
| 2663 |
+
os.makedirs(os.path.dirname(p), exist_ok=True)
|
| 2664 |
+
with io.open(p, "w", encoding="utf-8") as fh:
|
| 2665 |
+
json.dump(d, fh, indent=1)
|
| 2666 |
+
except OSError:
|
| 2667 |
+
pass
|
| 2668 |
+
|
| 2669 |
+
|
| 2670 |
+
def _mirror_lag_cycles(feed, now=None, cache_path=None, save=True):
|
| 2671 |
+
"""Update and return the per-(repo/copy) lag cycle counts from the feed. A 'cycle' is one DISTINCT feed
|
| 2672 |
+
reading (`checked`) during which a copy's head != the repo's target. The highway passes many times per feed
|
| 2673 |
+
reading; only a NEW `checked` advances a count, so N cycles is ~N feed intervals behind, not N highway passes
|
| 2674 |
+
(the A20/M34 trap: an alert whose threshold is really 'the loop ran again' goes off for nothing). Returns
|
| 2675 |
+
{"<repo>/<copy>": {cycles, since, last_checked, target}} for every copy currently behind; copies now caught up
|
| 2676 |
+
are dropped, and a changed target restarts the count (a fresh lag instance). Writes the cache unless save=False
|
| 2677 |
+
(the suite). Within one feed reading the update is idempotent, so the two lag detectors calling it in one pass
|
| 2678 |
+
cannot double-count."""
|
| 2679 |
+
cache = _load_lag_cache(cache_path)
|
| 2680 |
+
cur = str((feed or {}).get("checked") or "")
|
| 2681 |
+
repos = (feed or {}).get("repos") or {}
|
| 2682 |
+
if isinstance(repos, list):
|
| 2683 |
+
repos = {r.get("name"): r for r in repos if isinstance(r, dict) and r.get("name")}
|
| 2684 |
+
behind_now = {}
|
| 2685 |
+
for name, entry in (repos.items() if isinstance(repos, dict) else []):
|
| 2686 |
+
if not isinstance(entry, dict):
|
| 2687 |
+
continue
|
| 2688 |
+
target = str(entry.get("target") or "")
|
| 2689 |
+
heads = entry.get("heads") or {}
|
| 2690 |
+
if not target:
|
| 2691 |
+
continue
|
| 2692 |
+
for copy in UNISON_COPIES:
|
| 2693 |
+
head = str(heads.get(copy) or "")
|
| 2694 |
+
if not head or head == target:
|
| 2695 |
+
continue
|
| 2696 |
+
key = "%s/%s" % (name, copy)
|
| 2697 |
+
prev = cache.get(key) or {}
|
| 2698 |
+
if not prev or prev.get("target") != target:
|
| 2699 |
+
behind_now[key] = {"cycles": 1, "since": cur, "last_checked": cur, "target": target}
|
| 2700 |
+
elif cur and prev.get("last_checked") != cur:
|
| 2701 |
+
behind_now[key] = {"cycles": int(prev.get("cycles", 0)) + 1, "since": prev.get("since", cur),
|
| 2702 |
+
"last_checked": cur, "target": target}
|
| 2703 |
+
else:
|
| 2704 |
+
behind_now[key] = dict(prev) # same feed reading (or no `checked`) -> no new cycle
|
| 2705 |
+
if save:
|
| 2706 |
+
_save_lag_cache(behind_now, cache_path)
|
| 2707 |
+
return behind_now
|
| 2708 |
+
|
| 2709 |
+
|
| 2710 |
+
def detect_mirror_lag(health=None, feed=None, url=None, get=None, now=None, cache_path=None, save=True):
|
| 2711 |
+
"""A mirror copy the loop CAN reversibly re-sync -- Hugging Face or the website -- has stayed behind the unison
|
| 2712 |
+
target across more than MIRROR_LAG_CYCLES distinct feed readings: the task that owns it is firing, yet the copy
|
| 2713 |
+
is not catching up. Remedy AUTO_REVERSIBLE (trigger_mirror_sync): re-run the guarded, idempotent task that owns
|
| 2714 |
+
the lagging copy (CovenantHFMirror for hf, CovenantSiteAutoDeploy for the website). The GitHub copy is NOT here:
|
| 2715 |
+
a public push belongs to repo_autopush and is PROPOSE_ONLY (detect_github_push_lag).
|
| 2716 |
+
|
| 2717 |
+
PERSISTENCE, NOT A BLIP. A copy one reading behind is a mirror mid-catch-up, not a fault; only a copy behind
|
| 2718 |
+
across >N readings -- the owning task has had N chances and the copy still trails -- is the finding. UNKNOWN
|
| 2719 |
+
(never OK) when the feed is not ready, so a feed outage reads as unison_feed_stale's finding, not as 'no lag'."""
|
| 2720 |
+
feed, note = _resolve_feed(feed, url, get)
|
| 2721 |
+
if not isinstance(feed, dict) or feed.get("status") != "ok":
|
| 2722 |
+
return {"state": UNKNOWN, "measured": {"why": note or "feed not ready (no status ok)"}}
|
| 2723 |
+
lag = _mirror_lag_cycles(feed, now=now, cache_path=cache_path, save=save)
|
| 2724 |
+
laggards = {k: v for k, v in lag.items()
|
| 2725 |
+
if k.rsplit("/", 1)[-1] in ("hf", "website") and int(v.get("cycles", 0)) > MIRROR_LAG_CYCLES}
|
| 2726 |
+
rerun = sorted({COPY_TASK[k.rsplit("/", 1)[-1]] for k in laggards})
|
| 2727 |
+
return {"state": PRESENT if laggards else ABSENT,
|
| 2728 |
+
"measured": {"over_%d_cycles" % MIRROR_LAG_CYCLES: {k: v.get("cycles") for k, v in laggards.items()},
|
| 2729 |
+
"rerun_tasks": rerun, "cycles_needed": MIRROR_LAG_CYCLES}}
|
| 2730 |
+
|
| 2731 |
+
|
| 2732 |
+
def detect_github_push_lag(health=None, feed=None, url=None, get=None, now=None, cache_path=None, save=True):
|
| 2733 |
+
"""The GitHub copy of a repo has stayed behind the unison target across more than MIRROR_LAG_CYCLES feed
|
| 2734 |
+
readings. NO AUTO REMEDY HERE: pushing to a public GitHub repo is never AUTO_REVERSIBLE. It belongs to the
|
| 2735 |
+
repo_autopush pusher (CovenantRepoAutopush), itself gated on open-PRs, opsec and fast-forward, and whose
|
| 2736 |
+
arming is the operator's call. This SENSES the lag and names who owns the push; it never performs one."""
|
| 2737 |
+
feed, note = _resolve_feed(feed, url, get)
|
| 2738 |
+
if not isinstance(feed, dict) or feed.get("status") != "ok":
|
| 2739 |
+
return {"state": UNKNOWN, "measured": {"why": note or "feed not ready (no status ok)"}}
|
| 2740 |
+
lag = _mirror_lag_cycles(feed, now=now, cache_path=cache_path, save=save)
|
| 2741 |
+
behind = {k: int(v.get("cycles", 0)) for k, v in lag.items()
|
| 2742 |
+
if k.rsplit("/", 1)[-1] == "github" and int(v.get("cycles", 0)) > MIRROR_LAG_CYCLES}
|
| 2743 |
+
return {"state": PRESENT if behind else ABSENT,
|
| 2744 |
+
"measured": {"github_behind_over_%d_cycles" % MIRROR_LAG_CYCLES: behind,
|
| 2745 |
+
"owner": "repo_autopush (CovenantRepoAutopush) -- gated public push, PROPOSE_ONLY; never run here"}}
|
| 2746 |
+
|
| 2747 |
+
|
| 2748 |
DETECTORS = {
|
| 2749 |
"node_down": detect_node_down,
|
| 2750 |
"tetsu_cannot_answer": detect_tetsu_cannot_answer,
|
|
|
|
| 2768 |
"sweep_not_current": detect_sweep_not_current,
|
| 2769 |
"ambassador_stalled": detect_ambassador_stalled,
|
| 2770 |
"schedule_stalled": detect_schedule_stalled,
|
| 2771 |
+
# The sync fabric (A405, 2026-11-02): the unison feed and the operator-armed tasks that keep the repo
|
| 2772 |
+
# mirrors in step. Feed-stale and github-lag propose only; a present-but-stalled task and HF/website lag
|
| 2773 |
+
# are reversibly healed by re-running the already-armed, guarded, idempotent task that owns them.
|
| 2774 |
+
"unison_feed_stale": detect_unison_feed_stale,
|
| 2775 |
+
"sync_task_not_firing": detect_sync_task_not_firing,
|
| 2776 |
+
"sync_task_missing": detect_sync_task_missing,
|
| 2777 |
+
"mirror_lag": detect_mirror_lag,
|
| 2778 |
+
"github_push_lag": detect_github_push_lag,
|
| 2779 |
"moltbook_strikes": detect_moltbook_strikes,
|
| 2780 |
"threefold_witness": detect_threefold_witness,
|
| 2781 |
# "public_email_exposure" RETIRED the day it was added (A306), his words 2026-10-07: "they aren't issues idgaf".
|
|
|
|
| 3426 |
scanned, "; ".join(mendable))
|
| 3427 |
|
| 3428 |
|
| 3429 |
+
def _run_scheduled_task(name):
|
| 3430 |
+
"""(ok, detail) -- ask Windows to run an already-armed task NOW (schtasks /run). This starts the SAME guarded,
|
| 3431 |
+
idempotent run the operator's schedule already makes on its timer; it changes no task config and no file here."""
|
| 3432 |
+
import subprocess
|
| 3433 |
+
try:
|
| 3434 |
+
p = subprocess.run(["schtasks", "/run", "/tn", name], capture_output=True, text=True, timeout=60,
|
| 3435 |
+
creationflags=_NOWIN)
|
| 3436 |
+
return p.returncode == 0, ((p.stdout or "") + (p.stderr or "")).strip()[:200]
|
| 3437 |
+
except Exception as e: # noqa: BLE001
|
| 3438 |
+
return False, "%s: %s" % (type(e).__name__, e)
|
| 3439 |
+
|
| 3440 |
+
|
| 3441 |
+
def _rerun_named(names, dry_run):
|
| 3442 |
+
"""(ok, detail) for re-running a list of armed sync tasks. ok=False ('held', never counted as a failure) when
|
| 3443 |
+
there is nothing to re-run or every /run was refused; ok=True when at least one started."""
|
| 3444 |
+
names = list(names or [])
|
| 3445 |
+
if not names:
|
| 3446 |
+
return False, "no armed sync task to re-run"
|
| 3447 |
+
if dry_run:
|
| 3448 |
+
return True, "would re-run: " + ", ".join(names)
|
| 3449 |
+
ran, failed = [], []
|
| 3450 |
+
for n in names:
|
| 3451 |
+
ok, detail = _run_scheduled_task(n)
|
| 3452 |
+
(ran if ok else failed).append(n if ok else "%s (%s)" % (n, detail))
|
| 3453 |
+
if not ran:
|
| 3454 |
+
return False, "schtasks /run refused: " + "; ".join(failed)
|
| 3455 |
+
return True, "re-ran " + ", ".join(ran) + ("; failed: " + "; ".join(failed) if failed else "")
|
| 3456 |
+
|
| 3457 |
+
|
| 3458 |
+
def remedy_rerun_sync_task(measured, dry_run=True):
|
| 3459 |
+
"""STATELESS: re-run the armed, guarded, idempotent sync task(s) the detector found present-and-not-firing.
|
| 3460 |
+
This does only what the operator-armed schedule already does on its own timer -- it changes no config and no
|
| 3461 |
+
file here, and under a DRY/staged pass (A244) it runs nothing."""
|
| 3462 |
+
return _rerun_named((measured or {}).get("not_firing"), dry_run)
|
| 3463 |
+
|
| 3464 |
+
|
| 3465 |
+
def remedy_trigger_mirror_sync(measured, dry_run=True):
|
| 3466 |
+
"""STATELESS: re-run the guarded, idempotent task that owns a lagging HF/website mirror copy -- the same run
|
| 3467 |
+
its schedule makes. A public GitHub push is never here; that is repo_autopush's, and PROPOSE_ONLY."""
|
| 3468 |
+
return _rerun_named((measured or {}).get("rerun_tasks"), dry_run)
|
| 3469 |
+
|
| 3470 |
+
|
| 3471 |
+
def remedy_propose_unison_feed(measured, dry_run=True):
|
| 3472 |
+
"""PROPOSE_ONLY, never executed by the engine. The feed runs on TETSU; this machine cannot restart it, and
|
| 3473 |
+
reaching across to his machine to restart a service is not reversible from here. apply_remedy refuses the
|
| 3474 |
+
class and turns this into a proposal carrying the covenant's reading and the cost beside the gain."""
|
| 3475 |
+
return False, "the unison feed runs on TETSU; a person restarts or unsticks it there"
|
| 3476 |
+
|
| 3477 |
+
|
| 3478 |
REMEDIES = {
|
| 3479 |
+
"rerun_sync_task": {"fn": remedy_rerun_sync_task, "klass": AUTO_REVERSIBLE,
|
| 3480 |
+
"for": ["sync_task_not_firing"], "kind": "stateless",
|
| 3481 |
+
# Async, and graded by a later pass: schtasks /run returns at once while the task runs on,
|
| 3482 |
+
# and a run in flight reports result 0x41301 ('currently running'), which a synchronous
|
| 3483 |
+
# re-sense would read as still-not-firing and quarantine a remedy that works -- the exact
|
| 3484 |
+
# fetch_build / restart_nodes trap. The day's budget keeps a persistently-failing task
|
| 3485 |
+
# from being hammered every pass; a task the re-run cannot revive is graded 'did not fix'
|
| 3486 |
+
# and quarantined for a person, which is correct.
|
| 3487 |
+
"async_for": ["sync_task_not_firing"], "grade_after_s": 900.0, "cooldown_s": 1800,
|
| 3488 |
+
"touches": ["an already-armed, guarded sync task's next run (re-run only; the schedule and its config are untouched)"],
|
| 3489 |
+
"benefit": {"gains": ["a mirror task that stalled or last failed runs again without waiting for its timer or a person",
|
| 3490 |
+
"the sync fabric keeps moving"],
|
| 3491 |
+
"cost": ["one extra guarded, idempotent run (the task's own ff-only / opsec gates still apply); a task that keeps failing is quarantined for a person"],
|
| 3492 |
+
"irreversible": []}},
|
| 3493 |
+
"trigger_mirror_sync": {"fn": remedy_trigger_mirror_sync, "klass": AUTO_REVERSIBLE,
|
| 3494 |
+
"for": ["mirror_lag"], "kind": "stateless",
|
| 3495 |
+
# Two feed readings to catch up before grading (the feed moves every interval_s), same
|
| 3496 |
+
# async reasoning as rerun_sync_task.
|
| 3497 |
+
"async_for": ["mirror_lag"], "grade_after_s": 1800.0, "cooldown_s": 1800,
|
| 3498 |
+
"touches": ["the guarded sync task that owns a lagging HF/website copy (re-run only; no config change, no GitHub push)"],
|
| 3499 |
+
"benefit": {"gains": ["an HF or website copy stuck behind the unison target is re-synced by its own task, without a person"],
|
| 3500 |
+
"cost": ["one extra guarded, idempotent run; the github copy is never touched here"],
|
| 3501 |
+
"irreversible": []}},
|
| 3502 |
+
"propose_unison_feed": {"fn": remedy_propose_unison_feed, "klass": PROPOSE_ONLY,
|
| 3503 |
+
"for": ["unison_feed_stale"], "kind": "needs a person",
|
| 3504 |
+
"touches": ["the unison feed on TETSU"],
|
| 3505 |
+
"benefit": {"gains": ["the whole sync fabric stops being frozen by a stuck or dead feed",
|
| 3506 |
+
"downstream tasks stop reading a stale 'nothing due' as quiet"],
|
| 3507 |
+
"cost": ["a person's attention on TETSU to restart or unstick the feed"],
|
| 3508 |
+
"irreversible": []}},
|
| 3509 |
"refresh_defender": {"fn": remedy_refresh_defender, "klass": AUTO_REVERSIBLE,
|
| 3510 |
"for": ["defense_lapse"], "kind": "stateless",
|
| 3511 |
"touches": ["Defender's signature update, judged by covenant_immune.update(), and a scan of our own tree; never its settings"],
|
|
|
|
| 4538 |
# repairs it" without that is a nudge, not a finding.
|
| 4539 |
alerts.append("highway: %s is present and nothing here repairs it -- %s"
|
| 4540 |
% (name, json.dumps(c["measured"])[:400]))
|
| 4541 |
+
# A405 (2026-11-02, "loosen autonomy ... create its own lane"): the AUTO_GATED
|
| 4542 |
+
# auto-onboard lane runs as its OWN pass in covenant_onboard.py -- a third class
|
| 4543 |
+
# between AUTO_REVERSIBLE and PROPOSE_ONLY, gated on an operator-rooted roster and
|
| 4544 |
+
# a verified node identity. This hook is the whole of its footprint here: it does
|
| 4545 |
+
# not touch apply_remedy or the existing remedies. dry_run carries A244 (the
|
| 4546 |
+
# watchdog passes dry_run=not highway_may_act(), so a staged copy runs it DRY), and
|
| 4547 |
+
# a paused highway has already returned above, so the lane never acts when paused.
|
| 4548 |
+
# It SHIPS DARK: no roster and not armed, so this adds one "DARK" info line and
|
| 4549 |
+
# repairs nothing. Guarded -- a notice from a new lane must never fell the road.
|
| 4550 |
+
try:
|
| 4551 |
+
import covenant_onboard as _ob
|
| 4552 |
+
ob_alerts, ob_infos = _ob.run_once(dry_run=dry_run, ledger=ledger)
|
| 4553 |
+
alerts.extend(ob_alerts)
|
| 4554 |
+
infos.extend(ob_infos)
|
| 4555 |
+
except Exception as _e: # noqa: BLE001
|
| 4556 |
+
infos.append("highway: auto-onboard lane unavailable -- %s: %s" % (type(_e).__name__, _e))
|
| 4557 |
return alerts, infos
|
| 4558 |
|
| 4559 |
|
covenant_onboard.py
ADDED
|
@@ -0,0 +1,738 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""covenant_onboard.py -- the highway's AUTO_GATED lane: auto-onboard a KNOWN node.
|
| 3 |
+
|
| 4 |
+
ASKED 2026-11-02: the operator chose to "loosen autonomy" for the mycelial
|
| 5 |
+
highway and to "create its own lane." The highway's founding line (see
|
| 6 |
+
covenant_highway.py) is two classes: AUTO_REVERSIBLE may be run by the engine;
|
| 7 |
+
PROPOSE_ONLY -- config, install, money, rule, a narrowing of choice -- may never
|
| 8 |
+
be. Onboarding a node sat in PROPOSE_ONLY, so a device the operator already owns
|
| 9 |
+
could appear on the tailnet and the highway would only ever PROPOSE peering it,
|
| 10 |
+
never do it. This file adds a THIRD class between the two, and nothing else:
|
| 11 |
+
|
| 12 |
+
AUTO_GATED -- the engine MAY act, but only through a gate rooted in the
|
| 13 |
+
operator, and only on a reversible, local action.
|
| 14 |
+
|
| 15 |
+
WHY A SEPARATE MODULE, NOT A NEW BRANCH IN apply_remedy. The highway's
|
| 16 |
+
apply_remedy dispatches on one axis -- is the class AUTO_REVERSIBLE? -- and its
|
| 17 |
+
invariants (NEVER_AUTOMATIC, the operator's choices, stateless-or-undo) are
|
| 18 |
+
written for repairs of a sensed fault. AUTO_GATED is a different shape: its
|
| 19 |
+
"condition" is a roster the operator signed, its gate verifies an identity over
|
| 20 |
+
the network, and its actions peer or arm rather than repair. Folding that into
|
| 21 |
+
apply_remedy would widen the one function every repair flows through, on the
|
| 22 |
+
same day a sibling session is editing the detectors beside it. So the lane lives
|
| 23 |
+
here, with its own gate and its own ledger rows (written to the SAME highway
|
| 24 |
+
ledger, klass AUTO_GATED, so one audit sees everything), and the highway's hook
|
| 25 |
+
is a single call. apply_remedy is not touched; the existing autonomous execution
|
| 26 |
+
is unchanged.
|
| 27 |
+
|
| 28 |
+
THE GATE. The engine auto-acts ONLY when ALL of these hold (any miss -> it does
|
| 29 |
+
not act; it PROPOSES or REFUSES, and says why):
|
| 30 |
+
|
| 31 |
+
1. A model's output never triggers it. Data is not authority -- the same line
|
| 32 |
+
as the Jang-woo reply, and it binds our own fleet, the uncensored 27B
|
| 33 |
+
included. The trigger's source must be a roster scan or tailnet presence,
|
| 34 |
+
never a model.
|
| 35 |
+
2. ARMED. The operator wrote ops/AUTO_ONBOARD_ARMED choosing a trust root, AND
|
| 36 |
+
that trust root is satisfiable now. Absent either, the lane is DARK: it
|
| 37 |
+
senses and proposes, it acts on nothing. It ships DARK.
|
| 38 |
+
3. A roster the operator ROOTED. Either (a) ops/NODE_ROSTER.json signed by the
|
| 39 |
+
operator's OWN key and the signature verifies here, or (b) the roster is
|
| 40 |
+
committed to the repo and every entry verifies as owned by the operator's
|
| 41 |
+
own Tailscale account (whois). Strongest present wins; neither -> dark. The
|
| 42 |
+
highway only READS and VERIFIES the roster. It never writes it.
|
| 43 |
+
4. The target is ON that roster, and the action is one the roster allows for
|
| 44 |
+
it. An unknown node is never auto-onboarded.
|
| 45 |
+
5. Identity verified, not minted. `tailscale whois <ip>` must name the
|
| 46 |
+
operator's account and the roster's Machine ID, and the node key pinned
|
| 47 |
+
from `tailscale status --json` must equal the roster's. An IP alone is
|
| 48 |
+
never trusted. Keys are only ever verified here, never created or copied.
|
| 49 |
+
6. Reversible, LOCAL, exactly undoable. Auto-peer a known node (a line in
|
| 50 |
+
ops/local_peers.txt; un-peer undoes it) and auto-arm a vetted task that
|
| 51 |
+
keeps its OWN gate (disable undoes it). Anything that publishes, pushes,
|
| 52 |
+
posts or sends is NOT reversible and stays PROPOSE_ONLY.
|
| 53 |
+
7. Still PROPOSE_ONLY, forever: installing software; creating or granting keys
|
| 54 |
+
or signing authority; the consent roster; onboarding an unknown node; any
|
| 55 |
+
firewall / power / security / system-setting change on any node, even a
|
| 56 |
+
reversible one; anything touching Tetsu's key, consent or path; any change
|
| 57 |
+
to the roster itself, removals included.
|
| 58 |
+
8. A per-node refusal is honoured. A node may declare "no system-setting
|
| 59 |
+
auto-action" and the engine respects it whatever the class. TETSU declares
|
| 60 |
+
it (ops/node_refusals.json) -- its Claude session's standing line on that
|
| 61 |
+
machine.
|
| 62 |
+
9. The operator's choices (ops/OPERATOR_CHOICES.json) and the pause switch are
|
| 63 |
+
honoured, and the lane is DRY wherever highway_may_act() is False (a staged
|
| 64 |
+
copy or a fresh clone -- A244). Every auto-action is logged AND announced
|
| 65 |
+
with its undo command, rate-limited per node.
|
| 66 |
+
|
| 67 |
+
SOVEREIGNTY IS IMMOVABLE. Onboarding grants no signing or consent authority --
|
| 68 |
+
that is the owner's alone (Tetsu, 10-31: "a decision for the OWNER"). The host
|
| 69 |
+
must not be able to command the signer. Key_id ...dd885ef9... is Tetsu's own
|
| 70 |
+
CONSENT key, not a node key; nothing here gates on it, signs with it, or touches
|
| 71 |
+
it.
|
| 72 |
+
|
| 73 |
+
USE
|
| 74 |
+
python covenant_onboard.py # what the lane sees, and whether it is armed
|
| 75 |
+
python covenant_onboard.py --plan # the candidates and what the gate says of each (DRY)
|
| 76 |
+
LICENCE: public domain.
|
| 77 |
+
"""
|
| 78 |
+
from __future__ import annotations
|
| 79 |
+
|
| 80 |
+
import json
|
| 81 |
+
import os
|
| 82 |
+
import re
|
| 83 |
+
import subprocess
|
| 84 |
+
import time
|
| 85 |
+
|
| 86 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 87 |
+
_NOWIN = 0x08000000 if os.name == "nt" else 0
|
| 88 |
+
|
| 89 |
+
# The third class. A string, like the other two in covenant_highway.py, so a
|
| 90 |
+
# ledger row and a test can name it without importing the engine.
|
| 91 |
+
AUTO_GATED = "AUTO_GATED"
|
| 92 |
+
|
| 93 |
+
# Files the operator owns. The highway READS these; it writes none of them.
|
| 94 |
+
ROSTER = os.path.join(HERE, "ops", "NODE_ROSTER.json") # operator-signed / operator-committed
|
| 95 |
+
OPERATOR_PUBKEY = os.path.join(HERE, "ops", "operator_pubkey.pem") # for trust root (a)
|
| 96 |
+
ARMED_FILE = os.path.join(HERE, "ops", "AUTO_ONBOARD_ARMED") # the operator's arm + trust-root choice
|
| 97 |
+
# This file the highway MAY write, because a refusal only ever TIGHTENS -- it can
|
| 98 |
+
# forbid an action, never permit one. Revoking is easier than granting (the spec's
|
| 99 |
+
# own rule), and a per-node "do not touch my system settings" is a revocation.
|
| 100 |
+
NODE_REFUSALS = os.path.join(HERE, "ops", "node_refusals.json")
|
| 101 |
+
# The highway's own ledger and choices, reused so one audit sees the whole road.
|
| 102 |
+
LEDGER = os.path.join(HERE, "ops", "highway.jsonl")
|
| 103 |
+
CHOICES = os.path.join(HERE, "ops", "OPERATOR_CHOICES.json")
|
| 104 |
+
# off-box peers, one comma-separated line of host:port -- the same file
|
| 105 |
+
# covenant_watchdog._local_peers() and covenant_prod.bat read.
|
| 106 |
+
LOCAL_PEERS = os.path.join(HERE, "ops", "local_peers.txt")
|
| 107 |
+
ARMED_TASKS = os.path.join(HERE, "ops", "armed_tasks.json") # tasks this lane has armed; each undoable
|
| 108 |
+
|
| 109 |
+
NOTIFY_COOLDOWN_S = 3600 # one announcement per (node, action) an hour, so the lane cannot spam his phone
|
| 110 |
+
_PEER_RE = re.compile(r"[\w.\-]+:\d+") # one host:port, the watchdog's own grammar
|
| 111 |
+
|
| 112 |
+
# Decisions a gate pass can return.
|
| 113 |
+
ACT = "ACT" # armed, gated, reversible: the engine performs it
|
| 114 |
+
DRY = "DRY" # it WOULD act, but a pause / may_act==False / dry_run holds it
|
| 115 |
+
PROPOSE = "PROPOSE" # fails closed OR the action is PROPOSE_ONLY: a person decides
|
| 116 |
+
REFUSE = "REFUSE" # a hard line: a model trigger, a bad signature, a failed identity, a refusal flag
|
| 117 |
+
|
| 118 |
+
# The only trigger sources that may ever reach ACT. A model is NOT here, and an
|
| 119 |
+
# unrecognised source is treated as a model would be: data, not authority.
|
| 120 |
+
TRIGGER_SOURCES = frozenset({"roster_scan", "tailnet_presence", "operator"})
|
| 121 |
+
|
| 122 |
+
# Action classes that are PROPOSE_ONLY forever, matched against what an action
|
| 123 |
+
# declares it IS. These never become AUTO_GATED however rooted the roster, so a
|
| 124 |
+
# careless roster entry naming one of them cannot make the engine run it.
|
| 125 |
+
PROPOSE_ONLY_CLASSES = frozenset({
|
| 126 |
+
"install", "key", "sign", "consent", "roster-change", "onboard-unknown",
|
| 127 |
+
"system-setting", "firewall", "power", "security",
|
| 128 |
+
})
|
| 129 |
+
|
| 130 |
+
# Reuse the highway's subject list so the two lanes refuse the same subjects.
|
| 131 |
+
try: # pragma: no cover - a trivial import guard
|
| 132 |
+
import covenant_highway as _H
|
| 133 |
+
NEVER_AUTOMATIC = _H.NEVER_AUTOMATIC
|
| 134 |
+
except Exception: # noqa: BLE001
|
| 135 |
+
NEVER_AUTOMATIC = ("money", "holdings", "wallet", "trader", "venue", "order",
|
| 136 |
+
"rule", "principle", "judge", "seat", "model", "phone",
|
| 137 |
+
"consent", "key", "grid", "breaker", "relay", "feeder",
|
| 138 |
+
"substation", "setpoint", "load_shed", "actuat", "scada", "intertie")
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
# ---- the actions this lane can perform, each reversible and local ----------
|
| 142 |
+
#
|
| 143 |
+
# `reversible` False means outward (publish/push/post/send): the gate refuses it
|
| 144 |
+
# to PROPOSE before it is ever reached, so an outward action can never be
|
| 145 |
+
# AUTO_GATED even if a roster lists it. `cls` is matched against
|
| 146 |
+
# PROPOSE_ONLY_CLASSES. `touches` is matched against NEVER_AUTOMATIC, the same
|
| 147 |
+
# way apply_remedy matches a remedy's touches -- a label is one careless edit
|
| 148 |
+
# from wrong, so the subject is checked, not the class alone.
|
| 149 |
+
|
| 150 |
+
def _do_peer(target, dry_run):
|
| 151 |
+
"""Add the node's address to ops/local_peers.txt. LOCAL state, no outward
|
| 152 |
+
effect; un-peer (remove the line) undoes it exactly."""
|
| 153 |
+
addr = str((target or {}).get("peer_addr") or "")
|
| 154 |
+
if not _PEER_RE.fullmatch(addr):
|
| 155 |
+
return False, "no valid host:port peer address on the roster entry"
|
| 156 |
+
cur = _read_peers()
|
| 157 |
+
if addr in cur:
|
| 158 |
+
return False, "already peered: %s" % addr
|
| 159 |
+
if dry_run:
|
| 160 |
+
return True, "would add peer %s to ops/local_peers.txt" % addr
|
| 161 |
+
_write_peers(cur + [addr])
|
| 162 |
+
return True, "peered %s" % addr
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _undo_peer(target):
|
| 166 |
+
addr = str((target or {}).get("peer_addr") or "")
|
| 167 |
+
return "remove %s from ops/local_peers.txt (un-peer)" % addr
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _do_arm_task(target, dry_run):
|
| 171 |
+
"""Arm a vetted task by recording it in ops/armed_tasks.json. Allowed ONLY
|
| 172 |
+
if the task keeps its own gate (ff-only / opsec / dry); disabling it undoes
|
| 173 |
+
it. This lane never removes the task's gate and never runs the task -- it
|
| 174 |
+
flips a local 'may run' flag the task itself still gates behind."""
|
| 175 |
+
task = (target or {}).get("task") or {}
|
| 176 |
+
name = str(task.get("name") or "")
|
| 177 |
+
gate = str(task.get("gate") or "")
|
| 178 |
+
if not name:
|
| 179 |
+
return False, "no task name"
|
| 180 |
+
if gate not in ("ff-only", "opsec", "dry"):
|
| 181 |
+
return False, "task does not keep its own gate (ff-only/opsec/dry): %r" % gate
|
| 182 |
+
cur = _read_armed_tasks()
|
| 183 |
+
if name in cur:
|
| 184 |
+
return False, "already armed: %s" % name
|
| 185 |
+
if dry_run:
|
| 186 |
+
return True, "would arm task %s (keeps its %s gate)" % (name, gate)
|
| 187 |
+
cur[name] = {"gate": gate, "armed_at": time.strftime("%Y-%m-%dT%H:%M:%S%z")}
|
| 188 |
+
_write_json(ARMED_TASKS, cur)
|
| 189 |
+
return True, "armed %s (keeps its %s gate)" % (name, gate)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _undo_arm_task(target):
|
| 193 |
+
name = str(((target or {}).get("task") or {}).get("name") or "")
|
| 194 |
+
return "disable %s (remove it from ops/armed_tasks.json)" % name
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
ACTIONS = {
|
| 198 |
+
"peer": {"fn": _do_peer, "undo": _undo_peer, "reversible": True, "cls": "peer",
|
| 199 |
+
"touches": ["a node address in ops/local_peers.txt"]},
|
| 200 |
+
"arm_task": {"fn": _do_arm_task, "undo": _undo_arm_task, "reversible": True, "cls": "arm_task",
|
| 201 |
+
"touches": ["a vetted task's armed flag in ops/armed_tasks.json"]},
|
| 202 |
+
# Declared so the gate and the tests can name them; all refuse before acting.
|
| 203 |
+
"install": {"fn": None, "undo": None, "reversible": True, "cls": "install",
|
| 204 |
+
"touches": ["software on a node"]},
|
| 205 |
+
"set_firewall": {"fn": None, "undo": None, "reversible": True, "cls": "firewall",
|
| 206 |
+
"touches": ["a node's firewall (a system setting)"]},
|
| 207 |
+
"push": {"fn": None, "undo": None, "reversible": False, "cls": "publish",
|
| 208 |
+
"touches": ["an outward publish"]},
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
# Which per-node refusal a given action class trips. A node that refuses
|
| 212 |
+
# "system-setting" refuses firewall/power/security actions whatever their class.
|
| 213 |
+
_REFUSAL_OF_CLASS = {"firewall": "system-setting", "power": "system-setting",
|
| 214 |
+
"security": "system-setting", "system-setting": "system-setting"}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
# ----------------------------------------------------------------- readers
|
| 218 |
+
|
| 219 |
+
def _read_json(path, default):
|
| 220 |
+
try:
|
| 221 |
+
with open(path, encoding="utf-8") as fh:
|
| 222 |
+
return json.load(fh)
|
| 223 |
+
except (OSError, ValueError):
|
| 224 |
+
return default
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _write_json(path, obj):
|
| 228 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 229 |
+
tmp = path + ".tmp"
|
| 230 |
+
with open(tmp, "w", encoding="utf-8") as fh:
|
| 231 |
+
json.dump(obj, fh, ensure_ascii=False, indent=2, sort_keys=True)
|
| 232 |
+
os.replace(tmp, path)
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def _read_peers(path=None):
|
| 236 |
+
"""The current off-box peers as a list of host:port, read from the one line
|
| 237 |
+
covenant_watchdog also reads. A malformed line reads as empty (no guessed peer)."""
|
| 238 |
+
path = path or LOCAL_PEERS
|
| 239 |
+
try:
|
| 240 |
+
with open(path, encoding="utf-8") as fh:
|
| 241 |
+
s = fh.readline().strip().strip(",")
|
| 242 |
+
except OSError:
|
| 243 |
+
return []
|
| 244 |
+
if not s:
|
| 245 |
+
return []
|
| 246 |
+
parts = [p for p in s.split(",") if p]
|
| 247 |
+
return parts if all(_PEER_RE.fullmatch(p) for p in parts) else []
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _write_peers(peers, path=None):
|
| 251 |
+
path = path or LOCAL_PEERS
|
| 252 |
+
line = ",".join(peers)
|
| 253 |
+
if line and not re.fullmatch(r"[\w.\-]+:\d+(,[\w.\-]+:\d+)*", line):
|
| 254 |
+
raise ValueError("refusing to write a malformed peers line: %r" % line)
|
| 255 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 256 |
+
tmp = path + ".tmp"
|
| 257 |
+
with open(tmp, "w", encoding="utf-8") as fh:
|
| 258 |
+
fh.write(line + "\n")
|
| 259 |
+
os.replace(tmp, path)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def _read_armed_tasks(path=None):
|
| 263 |
+
d = _read_json(path or ARMED_TASKS, {})
|
| 264 |
+
return d if isinstance(d, dict) else {}
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _operator_choices(path=None):
|
| 268 |
+
d = _read_json(path or CHOICES, {})
|
| 269 |
+
return d if isinstance(d, dict) else {}
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _node_refusals(path=None):
|
| 273 |
+
"""{node_id: [refused classes]} -- refusals only tighten, so this file is
|
| 274 |
+
safe for the highway to carry. Absent -> no refusals."""
|
| 275 |
+
d = _read_json(path or NODE_REFUSALS, {})
|
| 276 |
+
out = {}
|
| 277 |
+
if isinstance(d, dict):
|
| 278 |
+
for k, v in d.items():
|
| 279 |
+
if isinstance(v, list):
|
| 280 |
+
out[str(k)] = [str(x) for x in v]
|
| 281 |
+
elif isinstance(v, dict) and isinstance(v.get("refuses"), list):
|
| 282 |
+
out[str(k)] = [str(x) for x in v["refuses"]]
|
| 283 |
+
return out
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
# ----------------------------------------------------------------- roster + trust root
|
| 287 |
+
|
| 288 |
+
def _canonical_payload(roster):
|
| 289 |
+
"""The bytes a signature covers: the roster without its own signature meta.
|
| 290 |
+
sort_keys so the operator's signer and this verifier agree byte for byte."""
|
| 291 |
+
body = {k: v for k, v in (roster or {}).items()
|
| 292 |
+
if k not in ("signature", "signed_by", "alg")}
|
| 293 |
+
return json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _verify_signature(roster, pubkey_pem):
|
| 297 |
+
"""True only if the roster carries a signature that `pubkey_pem` verifies over
|
| 298 |
+
its canonical payload. Any missing piece, any error, any mismatch -> False.
|
| 299 |
+
Supports whatever key type the operator chose (Ed25519 or RSA PEM); a key this
|
| 300 |
+
cannot load is a key it cannot verify, which is False, not a pass."""
|
| 301 |
+
sig_hex = (roster or {}).get("signature")
|
| 302 |
+
if not sig_hex or not pubkey_pem:
|
| 303 |
+
return False
|
| 304 |
+
try:
|
| 305 |
+
from cryptography.hazmat.primitives import hashes, serialization
|
| 306 |
+
from cryptography.hazmat.primitives.asymmetric import ed25519, padding
|
| 307 |
+
key = serialization.load_pem_public_key(pubkey_pem.encode()
|
| 308 |
+
if isinstance(pubkey_pem, str) else pubkey_pem)
|
| 309 |
+
sig = bytes.fromhex(sig_hex)
|
| 310 |
+
payload = _canonical_payload(roster)
|
| 311 |
+
if isinstance(key, ed25519.Ed25519PublicKey):
|
| 312 |
+
key.verify(sig, payload)
|
| 313 |
+
else: # RSA (or any key whose verify takes a padding+hash); PSS is what a signer should use
|
| 314 |
+
key.verify(sig, payload, padding.PSS(mgf=padding.MGF1(hashes.SHA256()),
|
| 315 |
+
salt_length=padding.PSS.MAX_LENGTH), hashes.SHA256())
|
| 316 |
+
return True
|
| 317 |
+
except Exception: # noqa: BLE001 -- unverifiable == not verified
|
| 318 |
+
return False
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def load_roster(path=None):
|
| 322 |
+
"""The roster dict, or {} when absent/unreadable. {} is the fail-closed shape:
|
| 323 |
+
no roster means no node is known, so nothing is ever auto-onboarded."""
|
| 324 |
+
d = _read_json(path or ROSTER, {})
|
| 325 |
+
return d if isinstance(d, dict) else {}
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def _operator_pubkey(path=None):
|
| 329 |
+
try:
|
| 330 |
+
with open(path or OPERATOR_PUBKEY, encoding="utf-8") as fh:
|
| 331 |
+
return fh.read()
|
| 332 |
+
except OSError:
|
| 333 |
+
return None
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def roster_trust_root(roster=None, pubkey=None, committed=None):
|
| 337 |
+
"""(root_name, reason). root_name is 'signature' | 'whois' | None.
|
| 338 |
+
|
| 339 |
+
Strongest present wins. 'signature' needs the operator's public key and a
|
| 340 |
+
signature over the roster that verifies. 'whois' needs the roster committed
|
| 341 |
+
to the repo (so its authenticity rides the repo) AND is then only as strong
|
| 342 |
+
as the per-entry whois done at identity time. None -> the lane is dark.
|
| 343 |
+
"""
|
| 344 |
+
roster = load_roster() if roster is None else roster
|
| 345 |
+
if not roster:
|
| 346 |
+
return None, "no roster on disk -- auto-onboard is dark"
|
| 347 |
+
pk = _operator_pubkey() if pubkey is None else pubkey
|
| 348 |
+
if pk and _verify_signature(roster, pk):
|
| 349 |
+
return "signature", "roster signed by the operator's key and the signature verifies"
|
| 350 |
+
if pk and roster.get("signature"):
|
| 351 |
+
return None, "a signature is present but does NOT verify against the operator's key -- fail closed"
|
| 352 |
+
committed = _roster_is_committed() if committed is None else committed
|
| 353 |
+
if committed:
|
| 354 |
+
return "whois", "no valid signature; roster is repo-committed, anchored per-entry by whois ownership"
|
| 355 |
+
return None, ("roster is present but neither signed-and-verified nor repo-committed -- "
|
| 356 |
+
"no trust root, fail closed")
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def _roster_is_committed(path=None):
|
| 360 |
+
"""True only if NODE_ROSTER.json is tracked by git (the whois anchor requires
|
| 361 |
+
the roster's authenticity to ride the repo, not a file anyone could drop in)."""
|
| 362 |
+
path = path or ROSTER
|
| 363 |
+
env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
|
| 364 |
+
try:
|
| 365 |
+
r = subprocess.run(["git", "-C", HERE, "ls-files", "--error-unmatch", os.path.relpath(path, HERE)],
|
| 366 |
+
capture_output=True, text=True, timeout=20, env=env, creationflags=_NOWIN)
|
| 367 |
+
return r.returncode == 0
|
| 368 |
+
except Exception: # noqa: BLE001
|
| 369 |
+
return False
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
# ----------------------------------------------------------------- identity (tailscale)
|
| 373 |
+
|
| 374 |
+
def _tailscale_whois(ip, whois_fn=None):
|
| 375 |
+
"""{'machine_id':.., 'user':..} for an IP, or None. Never trusts an IP alone:
|
| 376 |
+
the CALLER checks machine id and owning user against the roster. Any failure
|
| 377 |
+
-> None -> fail closed."""
|
| 378 |
+
if whois_fn is not None:
|
| 379 |
+
return whois_fn(ip)
|
| 380 |
+
try:
|
| 381 |
+
r = subprocess.run(["tailscale", "whois", "--json", str(ip)],
|
| 382 |
+
capture_output=True, text=True, timeout=20, creationflags=_NOWIN)
|
| 383 |
+
if r.returncode != 0:
|
| 384 |
+
return None
|
| 385 |
+
d = json.loads(r.stdout or "null")
|
| 386 |
+
node = (d or {}).get("Node") or {}
|
| 387 |
+
user = (d or {}).get("UserProfile") or {}
|
| 388 |
+
return {"machine_id": node.get("StableID") or node.get("ID"),
|
| 389 |
+
"user": user.get("LoginName") or user.get("ID"),
|
| 390 |
+
"key": node.get("Key")}
|
| 391 |
+
except Exception: # noqa: BLE001
|
| 392 |
+
return None
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def _tailscale_nodekey(ip, status_fn=None):
|
| 396 |
+
"""The node's public key pinned from `tailscale status --json`, or None."""
|
| 397 |
+
if status_fn is not None:
|
| 398 |
+
return status_fn(ip)
|
| 399 |
+
try:
|
| 400 |
+
r = subprocess.run(["tailscale", "status", "--json"],
|
| 401 |
+
capture_output=True, text=True, timeout=20, creationflags=_NOWIN)
|
| 402 |
+
if r.returncode != 0:
|
| 403 |
+
return None
|
| 404 |
+
d = json.loads(r.stdout or "null") or {}
|
| 405 |
+
for peer in list((d.get("Peer") or {}).values()) + [d.get("Self") or {}]:
|
| 406 |
+
for a in (peer.get("TailscaleIPs") or []):
|
| 407 |
+
if str(a) == str(ip):
|
| 408 |
+
return peer.get("PublicKey") or peer.get("Key")
|
| 409 |
+
return None
|
| 410 |
+
except Exception: # noqa: BLE001
|
| 411 |
+
return None
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def verify_identity(node_id, entry, operator_account, whois_fn=None, status_fn=None):
|
| 415 |
+
"""(ok, reason). ok only when whois names the operator's account AND the
|
| 416 |
+
roster's Machine ID, and the node key pinned from status equals the roster's.
|
| 417 |
+
An IP alone is never enough; any missing or mismatched piece is False."""
|
| 418 |
+
ip = entry.get("ip")
|
| 419 |
+
if not ip:
|
| 420 |
+
return False, "roster entry for %s has no ip" % node_id
|
| 421 |
+
who = _tailscale_whois(ip, whois_fn=whois_fn)
|
| 422 |
+
if not who:
|
| 423 |
+
return False, "whois failed for %s (%s) -- fail closed" % (node_id, ip)
|
| 424 |
+
if operator_account and str(who.get("user")) != str(operator_account):
|
| 425 |
+
return False, "whois owner %r != operator %r" % (who.get("user"), operator_account)
|
| 426 |
+
if entry.get("machine_id") and str(who.get("machine_id")) != str(entry["machine_id"]):
|
| 427 |
+
return False, "whois machine id %r != rostered %r" % (who.get("machine_id"), entry["machine_id"])
|
| 428 |
+
pinned = _tailscale_nodekey(ip, status_fn=status_fn)
|
| 429 |
+
if entry.get("pubkey"):
|
| 430 |
+
if not pinned:
|
| 431 |
+
return False, "could not pin a node key for %s -- fail closed" % node_id
|
| 432 |
+
if str(pinned) != str(entry["pubkey"]):
|
| 433 |
+
return False, "pinned node key != rostered node key for %s" % node_id
|
| 434 |
+
return True, "identity verified by whois (owner, machine id) and pinned node key"
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
# ----------------------------------------------------------------- the gate
|
| 438 |
+
|
| 439 |
+
def _grid_route(action):
|
| 440 |
+
"""A reason string if `action` is a grid/ICS effect that must go to the grid
|
| 441 |
+
guard's together-vote, else None. The class is DERIVED from covenant_grid_guard's
|
| 442 |
+
ACTION_CLASS allowlist -- the one source of truth, reused not forked. If the guard
|
| 443 |
+
cannot be imported, this returns None: the known-action check still refuses
|
| 444 |
+
anything that is not one of this lane's own (non-grid) reversible actions, so a
|
| 445 |
+
missing guard opens no hole."""
|
| 446 |
+
try:
|
| 447 |
+
import covenant_grid_guard as G
|
| 448 |
+
cls = G.required_class(action)
|
| 449 |
+
except Exception: # noqa: BLE001
|
| 450 |
+
return None
|
| 451 |
+
if cls is not None:
|
| 452 |
+
holders = "+".join(getattr(G, "AUTHORITY_ROLES", ("operator", "tetsu")))
|
| 453 |
+
return ("grid/ICS action %r (class %s) -> covenant_grid_guard together-vote (%s), "
|
| 454 |
+
"fail-closed, never AUTO_GATED (data/anomaly is ALARM-only, never authority)"
|
| 455 |
+
% (action, cls, holders))
|
| 456 |
+
return None
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def is_armed(roster=None, armed_cfg=None, pubkey=None, committed=None):
|
| 460 |
+
"""(armed, trust_root, reason). The lane is armed ONLY when the operator wrote
|
| 461 |
+
ops/AUTO_ONBOARD_ARMED choosing a trust root AND that trust root is satisfiable
|
| 462 |
+
now. Ships DARK: the file does not exist, so this returns (False, None, ...)."""
|
| 463 |
+
cfg = _read_json(ARMED_FILE, None) if armed_cfg is None else armed_cfg
|
| 464 |
+
if not isinstance(cfg, dict) or not cfg.get("armed"):
|
| 465 |
+
return False, None, "ops/AUTO_ONBOARD_ARMED absent or armed!=true -- the lane is dark"
|
| 466 |
+
want = str(cfg.get("trust_root") or "auto")
|
| 467 |
+
root, why = roster_trust_root(roster=roster, pubkey=pubkey, committed=committed)
|
| 468 |
+
if root is None:
|
| 469 |
+
return False, None, "armed, but %s" % why
|
| 470 |
+
if want not in ("auto", root):
|
| 471 |
+
return False, None, "operator asked trust root %r but the strongest present is %r" % (want, root)
|
| 472 |
+
return True, root, "armed; trust root %s (%s)" % (root, why)
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def evaluate(action, target, trigger, *, roster=None, choices=None, refusals=None,
|
| 476 |
+
operator_account=None, armed_cfg=None, pubkey=None, committed=None,
|
| 477 |
+
whois_fn=None, status_fn=None, paused=None, may_act=True, now=None):
|
| 478 |
+
"""The whole gate for one (action, target, trigger), in order, fail-closed.
|
| 479 |
+
|
| 480 |
+
Returns {decision, reason, undo, action, node, klass}. decision is ACT only
|
| 481 |
+
when every gate passes AND the lane is armed AND nothing holds it dry; DRY
|
| 482 |
+
when it would act but a pause / may_act False / a declared dry holds it;
|
| 483 |
+
PROPOSE when it fails closed or the action is PROPOSE_ONLY; REFUSE on a hard
|
| 484 |
+
line (a model trigger, a bad signature, a failed identity, a refusal flag).
|
| 485 |
+
"""
|
| 486 |
+
node_id = str((target or {}).get("node") or (target or {}).get("id") or "?")
|
| 487 |
+
spec = ACTIONS.get(action)
|
| 488 |
+
base = {"action": action, "node": node_id, "klass": AUTO_GATED, "undo": "n/a"}
|
| 489 |
+
|
| 490 |
+
def out(decision, reason):
|
| 491 |
+
return dict(base, decision=decision, reason=reason)
|
| 492 |
+
|
| 493 |
+
# 1. A MODEL'S OUTPUT NEVER TRIGGERS AUTO_GATED. Data is not authority.
|
| 494 |
+
src = (trigger or {}).get("source") if isinstance(trigger, dict) else None
|
| 495 |
+
if src not in TRIGGER_SOURCES:
|
| 496 |
+
return out(REFUSE, "trigger source %r is not operator-rooted -- a model's output can never "
|
| 497 |
+
"trigger AUTO_GATED (data is not authority)" % src)
|
| 498 |
+
|
| 499 |
+
# 2. GRID / ICS DEFERS TO covenant_grid_guard -- BEFORE the known-action check, so a
|
| 500 |
+
# grid action named on a roster is routed to the guard rather than called "unknown".
|
| 501 |
+
# The class is DERIVED from the grid guard's own ACTION_CLASS allowlist (the single
|
| 502 |
+
# source of truth -- no forked grid word list, per the PG1/rule-6 lesson): any action
|
| 503 |
+
# it recognises is grid-domain and goes to the together-vote (operator + Tetsu),
|
| 504 |
+
# fail-closed, never AUTO_GATED. An action the allowlist does not know is not treated
|
| 505 |
+
# as grid here; it falls to the known-action check below, which refuses the unknown.
|
| 506 |
+
gr = _grid_route(action)
|
| 507 |
+
if gr:
|
| 508 |
+
return dict(base, decision=REFUSE, reason=gr, route="grid_guard")
|
| 509 |
+
|
| 510 |
+
# 3. a known action
|
| 511 |
+
if spec is None:
|
| 512 |
+
return out(REFUSE, "unknown action %r" % action)
|
| 513 |
+
|
| 514 |
+
# 4. a per-node refusal, honoured whatever the class and BEFORE the
|
| 515 |
+
# PROPOSE_ONLY short-circuit below, so it is a guard in its own right: a
|
| 516 |
+
# node that refuses 'system-setting' REFUSES (a hard no), where an
|
| 517 |
+
# unflagged node's system-setting action merely PROPOSES. TETSU sets this.
|
| 518 |
+
refusals = _node_refusals() if refusals is None else refusals
|
| 519 |
+
refused = set(refusals.get(node_id, []))
|
| 520 |
+
cls_refusal = _REFUSAL_OF_CLASS.get(spec.get("cls"))
|
| 521 |
+
if spec.get("cls") in refused or (cls_refusal and cls_refusal in refused):
|
| 522 |
+
return out(REFUSE, "%s refuses %s auto-actions (per-node refusal)" % (node_id, cls_refusal or spec["cls"]))
|
| 523 |
+
|
| 524 |
+
# 5. PROPOSE_ONLY forever -- by the action's declared class, and by its subject
|
| 525 |
+
if spec.get("cls") in PROPOSE_ONLY_CLASSES:
|
| 526 |
+
return out(PROPOSE, "action class %r is PROPOSE_ONLY forever -- a person decides" % spec["cls"])
|
| 527 |
+
crossed = sorted({w for w in NEVER_AUTOMATIC for t in spec.get("touches", []) if w in t.lower()})
|
| 528 |
+
if crossed:
|
| 529 |
+
return out(PROPOSE, "touches %s -- refused whatever class it claims" % ", ".join(crossed))
|
| 530 |
+
|
| 531 |
+
# 6. outward actions are not reversible, so never AUTO_GATED
|
| 532 |
+
if not spec.get("reversible"):
|
| 533 |
+
return out(PROPOSE, "action %r is not reversible (outward: cannot be unsent) -- PROPOSE_ONLY" % action)
|
| 534 |
+
|
| 535 |
+
# 7. ARMED and a trust root present, else DARK (fail closed -> propose)
|
| 536 |
+
armed, root, why = is_armed(roster=roster, armed_cfg=armed_cfg, pubkey=pubkey, committed=committed)
|
| 537 |
+
if not armed:
|
| 538 |
+
return out(PROPOSE, "auto-onboard dark: %s" % why)
|
| 539 |
+
base["root"] = root
|
| 540 |
+
|
| 541 |
+
# 8. the target is on the roster (an unknown node is never auto-onboarded)
|
| 542 |
+
r = load_roster() if roster is None else roster
|
| 543 |
+
nodes = (r.get("nodes") or {}) if isinstance(r, dict) else {}
|
| 544 |
+
entry = nodes.get(node_id)
|
| 545 |
+
if not isinstance(entry, dict):
|
| 546 |
+
return out(PROPOSE, "node %r is not on the roster -- onboarding an unknown node is PROPOSE_ONLY" % node_id)
|
| 547 |
+
|
| 548 |
+
# 9. the roster allows THIS action for THIS node
|
| 549 |
+
allowed = entry.get("allowed") or entry.get("actions") or []
|
| 550 |
+
if action not in allowed:
|
| 551 |
+
return out(REFUSE, "the roster does not allow %r for %s (allowed: %s)" % (action, node_id, allowed))
|
| 552 |
+
|
| 553 |
+
# 10. identity verified, not minted
|
| 554 |
+
acct = operator_account if operator_account is not None else r.get("operator_account")
|
| 555 |
+
ok, idreason = verify_identity(node_id, entry, acct, whois_fn=whois_fn, status_fn=status_fn)
|
| 556 |
+
if not ok:
|
| 557 |
+
return out(REFUSE, "identity not verified: %s" % idreason)
|
| 558 |
+
|
| 559 |
+
# 11. the operator's explicit choices are untouchable
|
| 560 |
+
choices = _operator_choices() if choices is None else choices
|
| 561 |
+
clash = sorted(set(spec.get("touches", [])) & set(choices or {}))
|
| 562 |
+
if node_id in (choices or {}):
|
| 563 |
+
clash = sorted(set(clash) | {node_id})
|
| 564 |
+
if clash:
|
| 565 |
+
return out(REFUSE, "the operator chose %s: %s" % (clash[0], (choices or {}).get(clash[0], "")))
|
| 566 |
+
|
| 567 |
+
# the undo is on record from here
|
| 568 |
+
undo_fn = spec.get("undo")
|
| 569 |
+
base["undo"] = undo_fn(target) if callable(undo_fn) else "n/a"
|
| 570 |
+
|
| 571 |
+
# 12. pause / may_act==False -> it would act, but DRY (A244)
|
| 572 |
+
if paused is None:
|
| 573 |
+
paused = _paused()
|
| 574 |
+
if paused or not may_act:
|
| 575 |
+
return out(DRY, "gated and armed, but held DRY (%s)" % ("paused" if paused else "highway_may_act() is False"))
|
| 576 |
+
|
| 577 |
+
return out(ACT, "armed, rostered, identity-verified, reversible -- the engine may act (%s)" % idreason)
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
def _paused():
|
| 581 |
+
try:
|
| 582 |
+
import covenant_pause as _p
|
| 583 |
+
is_paused, _why = _p.paused("highway")
|
| 584 |
+
return bool(is_paused)
|
| 585 |
+
except Exception: # noqa: BLE001
|
| 586 |
+
return False
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
# ----------------------------------------------------------------- announce + ledger
|
| 590 |
+
|
| 591 |
+
def _recent_notify(node_id, action, ledger=None, now=None):
|
| 592 |
+
now = time.time() if now is None else now
|
| 593 |
+
for row in reversed(_read_ledger(ledger)):
|
| 594 |
+
if row.get("klass") == AUTO_GATED and row.get("node") == node_id and row.get("action") == action \
|
| 595 |
+
and row.get("announced"):
|
| 596 |
+
return (now - float(row.get("at", 0))) < NOTIFY_COOLDOWN_S
|
| 597 |
+
return False
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
def announce(decision_row, ledger=None, notify_fn=None, now=None):
|
| 601 |
+
"""Announce an auto-action with its undo command, rate-limited per (node, action).
|
| 602 |
+
A notifier that can bring down what it reports on has inverted its purpose, so
|
| 603 |
+
this never raises and returns whether it sent."""
|
| 604 |
+
node_id, action = decision_row.get("node"), decision_row.get("action")
|
| 605 |
+
if _recent_notify(node_id, action, ledger=ledger, now=now):
|
| 606 |
+
return False
|
| 607 |
+
title = "highway auto-onboard: %s %s" % (action, node_id)
|
| 608 |
+
body = "%s\nundo: %s" % (decision_row.get("reason", ""), decision_row.get("undo", "n/a"))
|
| 609 |
+
try:
|
| 610 |
+
if notify_fn is not None:
|
| 611 |
+
notify_fn(title, body)
|
| 612 |
+
else:
|
| 613 |
+
import covenant_notify
|
| 614 |
+
covenant_notify.notify(title, body, priority="default")
|
| 615 |
+
return True
|
| 616 |
+
except Exception: # noqa: BLE001
|
| 617 |
+
return False
|
| 618 |
+
|
| 619 |
+
|
| 620 |
+
def _read_ledger(path=None):
|
| 621 |
+
out = []
|
| 622 |
+
try:
|
| 623 |
+
with open(path or LEDGER, encoding="utf-8") as fh:
|
| 624 |
+
for line in fh:
|
| 625 |
+
try:
|
| 626 |
+
out.append(json.loads(line))
|
| 627 |
+
except ValueError:
|
| 628 |
+
continue
|
| 629 |
+
except OSError:
|
| 630 |
+
pass
|
| 631 |
+
return out
|
| 632 |
+
|
| 633 |
+
|
| 634 |
+
def _write_ledger(row, path=None):
|
| 635 |
+
path = path or LEDGER
|
| 636 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 637 |
+
with open(path, "a", encoding="utf-8") as fh:
|
| 638 |
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
| 639 |
+
return row
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def act(action, target, trigger, *, dry_run=True, ledger=None, notify_fn=None, now=None, **gate):
|
| 643 |
+
"""Evaluate the gate and, only on ACT and not dry_run, perform the reversible
|
| 644 |
+
action -- then log the row (klass AUTO_GATED, to the highway ledger) and
|
| 645 |
+
announce it with its undo. DRY/PROPOSE/REFUSE are logged too: a refusal nobody
|
| 646 |
+
can see is indistinguishable from a bug that swallowed the work."""
|
| 647 |
+
d = evaluate(action, target, trigger, **gate)
|
| 648 |
+
row = {"t": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "at": round(now if now is not None else time.time(), 1),
|
| 649 |
+
"klass": AUTO_GATED, "action": action, "node": d.get("node"), "detector": "auto_onboard",
|
| 650 |
+
"decision": d["decision"], "why": d["reason"], "undo": d.get("undo", "n/a"),
|
| 651 |
+
"trigger_source": (trigger or {}).get("source") if isinstance(trigger, dict) else None,
|
| 652 |
+
"dry_run": bool(dry_run)}
|
| 653 |
+
if d["decision"] != ACT or dry_run:
|
| 654 |
+
row["outcome"] = {ACT: "dry run", DRY: "held (dry)", PROPOSE: "proposed", REFUSE: "refused"}[d["decision"]]
|
| 655 |
+
row["ran"] = False
|
| 656 |
+
return _write_ledger(row, ledger)
|
| 657 |
+
spec = ACTIONS.get(action) or {}
|
| 658 |
+
fn = spec.get("fn")
|
| 659 |
+
if not callable(fn):
|
| 660 |
+
row.update(outcome="refused", ran=False, why="no executable function for %r" % action)
|
| 661 |
+
return _write_ledger(row, ledger)
|
| 662 |
+
ok, detail = fn(target, dry_run=False)
|
| 663 |
+
row.update(ran=bool(ok), detail=str(detail)[:300],
|
| 664 |
+
outcome="done" if ok else "held", undo=(spec["undo"](target) if ok and callable(spec.get("undo")) else row["undo"]))
|
| 665 |
+
_write_ledger(row, ledger)
|
| 666 |
+
if ok:
|
| 667 |
+
row_sent = dict(d, undo=row["undo"])
|
| 668 |
+
if announce(row_sent, ledger=ledger, notify_fn=notify_fn, now=now):
|
| 669 |
+
_write_ledger(dict(row, announced=True, outcome="announced",
|
| 670 |
+
at=round(now if now is not None else time.time(), 1)), ledger)
|
| 671 |
+
return row
|
| 672 |
+
|
| 673 |
+
|
| 674 |
+
# ----------------------------------------------------------------- the pass
|
| 675 |
+
|
| 676 |
+
def _candidates(roster):
|
| 677 |
+
"""[(action, target, trigger)] the roster offers: a rostered node not yet
|
| 678 |
+
peered -> peer it; a vetted task not yet armed -> arm it. The trigger source
|
| 679 |
+
is 'roster_scan' -- never a model. An empty/absent roster yields nothing."""
|
| 680 |
+
out = []
|
| 681 |
+
nodes = (roster.get("nodes") or {}) if isinstance(roster, dict) else {}
|
| 682 |
+
peers = set(_read_peers())
|
| 683 |
+
for node_id, entry in sorted(nodes.items()):
|
| 684 |
+
if not isinstance(entry, dict):
|
| 685 |
+
continue
|
| 686 |
+
allowed = entry.get("allowed") or entry.get("actions") or []
|
| 687 |
+
if "peer" in allowed and str(entry.get("peer_addr") or "") not in peers:
|
| 688 |
+
out.append(("peer", {"node": node_id, "peer_addr": entry.get("peer_addr")},
|
| 689 |
+
{"source": "roster_scan"}))
|
| 690 |
+
for task in (entry.get("tasks") or []):
|
| 691 |
+
if "arm_task" in allowed and task.get("name") not in _read_armed_tasks():
|
| 692 |
+
out.append(("arm_task", {"node": node_id, "task": task}, {"source": "roster_scan"}))
|
| 693 |
+
return out
|
| 694 |
+
|
| 695 |
+
|
| 696 |
+
def run_once(dry_run=True, ledger=None, now=None, **gate):
|
| 697 |
+
"""One auto-onboard pass, in the highway's (alerts, infos) shape so its hook
|
| 698 |
+
can merge the lines. Honours pause and may_act (both default from the gate).
|
| 699 |
+
DARK and empty by default: no roster, not armed -> one info line, no action."""
|
| 700 |
+
alerts, infos = [], []
|
| 701 |
+
roster = gate.get("roster")
|
| 702 |
+
roster = load_roster() if roster is None else roster
|
| 703 |
+
armed, root, why = is_armed(roster=roster, pubkey=gate.get("pubkey"),
|
| 704 |
+
armed_cfg=gate.get("armed_cfg"), committed=gate.get("committed"))
|
| 705 |
+
if not armed:
|
| 706 |
+
infos.append("highway auto-onboard: DARK -- %s" % why)
|
| 707 |
+
return alerts, infos
|
| 708 |
+
infos.append("highway auto-onboard: armed (trust root %s)" % root)
|
| 709 |
+
for action, target, trigger in _candidates(roster):
|
| 710 |
+
row = act(action, target, trigger, dry_run=dry_run, ledger=ledger, now=now, roster=roster, **{
|
| 711 |
+
k: v for k, v in gate.items() if k in
|
| 712 |
+
("choices", "refusals", "operator_account", "armed_cfg", "pubkey", "committed",
|
| 713 |
+
"whois_fn", "status_fn", "paused", "may_act")})
|
| 714 |
+
line = "highway auto-onboard: %s %s -> %s (%s)" % (action, target.get("node"), row.get("outcome"), row.get("why", "")[:120])
|
| 715 |
+
(infos if row.get("outcome") in ("done", "announced", "dry run") else alerts).append(line)
|
| 716 |
+
return alerts, infos
|
| 717 |
+
|
| 718 |
+
|
| 719 |
+
# ----------------------------------------------------------------- CLI
|
| 720 |
+
|
| 721 |
+
def main(argv=None):
|
| 722 |
+
import argparse
|
| 723 |
+
ap = argparse.ArgumentParser(description="the highway's AUTO_GATED auto-onboard lane")
|
| 724 |
+
ap.add_argument("--plan", action="store_true", help="show the candidates and the gate's verdict (DRY)")
|
| 725 |
+
args = ap.parse_args(argv)
|
| 726 |
+
roster = load_roster()
|
| 727 |
+
armed, root, why = is_armed(roster=roster)
|
| 728 |
+
print("auto-onboard: %s" % ("ARMED (trust root %s)" % root if armed else "DARK -- %s" % why))
|
| 729 |
+
if args.plan:
|
| 730 |
+
alerts, infos = run_once(dry_run=True)
|
| 731 |
+
for ln in infos + alerts:
|
| 732 |
+
print(" " + ln)
|
| 733 |
+
return 0
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
if __name__ == "__main__":
|
| 737 |
+
import sys
|
| 738 |
+
sys.exit(main())
|
covenant_one.py
CHANGED
|
@@ -619,10 +619,22 @@ SUITES = [
|
|
| 619 |
# mutation-tested. Hermetic: fixture conditions, temp ledgers, no node and
|
| 620 |
# no network, so it measures the rules rather than the machine.
|
| 621 |
("test_h1_highway.py", 180, "DAILY + GUARDS"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 622 |
# NA1 (2026-10-20): H1i names six protected subjects; the engine's NEVER_AUTOMATIC tuple had fourteen
|
| 623 |
# (twenty-four since 2026-10-30: the grid/ICS-OT actuation words, A392).
|
| 624 |
# NA1 drives every one through apply_remedy so the money/seat/rule/consent/key list cannot silently shrink.
|
| 625 |
("test_na1_never_automatic.py", 120, "DAILY + GUARDS"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 626 |
# PB1 (2026-09-26, A231): the highway's two phone-build signals measure what they name -- a look that finds
|
| 627 |
# nothing newer counts as a look (fetch_build was quarantined for it), and "behind" means behind in the
|
| 628 |
# files the APK ships, not in every docs commit.
|
|
@@ -824,6 +836,12 @@ SUITES = [
|
|
| 824 |
# HM1 (2026-10-31, his "fix hf_mirror.py"): the HF mirror pushes fast-forward only, behind his opsec scan, and
|
| 825 |
# snapshot-only repos never send history -- it used to force-push full history from a hook-less clone (A396).
|
| 826 |
("test_hm1_hf_mirror.py", 120, "SECURITY"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 827 |
("test_sw1_study_web.py", 120, "JUDGE"),
|
| 828 |
("test_r2v_frames.py", 240, "JUDGE"),
|
| 829 |
# PC2 (2026-10-03, the core's pin; 2026-10-06, A315, the other four through tools/pin_deploy.py): every
|
|
|
|
| 619 |
# mutation-tested. Hermetic: fixture conditions, temp ledgers, no node and
|
| 620 |
# no network, so it measures the rules rather than the machine.
|
| 621 |
("test_h1_highway.py", 180, "DAILY + GUARDS"),
|
| 622 |
+
# SF1 (2026-11-02, A405): the highway's sync-fabric detectors and remedies, each driven both ways. The
|
| 623 |
+
# unison feed's staleness, a sync task that fails or stops firing (re-run, AUTO_REVERSIBLE), a task gone
|
| 624 |
+
# entirely (config, PROPOSE_ONLY), and HF/website vs github mirror lag -- github never auto-pushed. Hermetic:
|
| 625 |
+
# feed and scheduler are fixtures, the cache a temp file, no node and no network.
|
| 626 |
+
("test_sf1_sync_fabric.py", 120, "DAILY + GUARDS"),
|
| 627 |
# NA1 (2026-10-20): H1i names six protected subjects; the engine's NEVER_AUTOMATIC tuple had fourteen
|
| 628 |
# (twenty-four since 2026-10-30: the grid/ICS-OT actuation words, A392).
|
| 629 |
# NA1 drives every one through apply_remedy so the money/seat/rule/consent/key list cannot silently shrink.
|
| 630 |
("test_na1_never_automatic.py", 120, "DAILY + GUARDS"),
|
| 631 |
+
# OG1 (2026-11-02, "loosen autonomy ... create its own lane"): the highway's AUTO_GATED auto-onboard lane
|
| 632 |
+
# (covenant_onboard.py) -- a third class between AUTO_REVERSIBLE and PROPOSE_ONLY. Every gate driven both
|
| 633 |
+
# ways: a model trigger refused, an off-roster node never onboarded, an absent/unsigned/bad-signature roster
|
| 634 |
+
# dark, a failed whois refused, a verified node auto-peered AND un-peered, a system-setting action refused on
|
| 635 |
+
# a flagged node, an outward action never AUTO_GATED. Hermetic: an in-process key signs a fixture roster,
|
| 636 |
+
# identity is stubbed, every path is a temp file. Ships DARK and NOT ARMED.
|
| 637 |
+
("test_og1_onboard.py", 120, "DAILY + GUARDS"),
|
| 638 |
# PB1 (2026-09-26, A231): the highway's two phone-build signals measure what they name -- a look that finds
|
| 639 |
# nothing newer counts as a look (fetch_build was quarantined for it), and "behind" means behind in the
|
| 640 |
# files the APK ships, not in every docs commit.
|
|
|
|
| 836 |
# HM1 (2026-10-31, his "fix hf_mirror.py"): the HF mirror pushes fast-forward only, behind his opsec scan, and
|
| 837 |
# snapshot-only repos never send history -- it used to force-push full history from a hook-less clone (A396).
|
| 838 |
("test_hm1_hf_mirror.py", 120, "SECURITY"),
|
| 839 |
+
# AP1/HD1 (2026-11-02, A404, his "Enable auto-push" + "as much redundancy as possible"): the laptop pusher
|
| 840 |
+
# off TETSU's unison feed (ff-only vs current GitHub main, open-PR guard, no --force/tags/unify-branch, never
|
| 841 |
+
# a snapshot repo's history) and hf_mirror's fail-closed holds-drift guard (refuse a history push when the
|
| 842 |
+
# feed can't confirm a repo was not marked HF-snapshot elsewhere). Both ways; each guard mutated -> suite RED.
|
| 843 |
+
("test_ap1_repo_autopush.py", 120, "SECURITY"),
|
| 844 |
+
("test_hd1_hf_mirror_holds.py", 60, "SECURITY"),
|
| 845 |
("test_sw1_study_web.py", 120, "JUDGE"),
|
| 846 |
("test_r2v_frames.py", 240, "JUDGE"),
|
| 847 |
# PC2 (2026-10-03, the core's pin; 2026-10-06, A315, the other four through tools/pin_deploy.py): every
|