Spaces:
Running
Running
Download server/fault_verifier.py from brand0nwalls/NetOps: direct link, hf CLI and curl.
- Browser
- Download file 32.6 kB
-
https://huggingface.co/spaces/brand0nwalls/NetOps/resolve/main/server/fault_verifier.py
- Command line
-
hf download hf://spaces/brand0nwalls/NetOps/server/fault_verifier.py
-
curl -L -o fault_verifier.py https://huggingface.co/spaces/brand0nwalls/NetOps/resolve/main/server/fault_verifier.py
32.6 kB
| """Fault-specific verification: check whether an injected fault has been fixed. | |
| Each fault type has a verifier that checks the specific condition was undone. | |
| Uses telemetry/cmd queries — works with both Mininet and Python sim. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from dataclasses import dataclass, field | |
| from server.constants import GroundTruth | |
| from server import telemetry | |
| class VerificationResult: | |
| fixed: bool | |
| checks: list[str] = field(default_factory=list) | |
| failures: list[str] = field(default_factory=list) | |
| def partial_score(self) -> float: | |
| if not self.checks: | |
| return 1.0 if self.fixed else 0.0 | |
| passed = len(self.checks) - len(self.failures) | |
| return passed / len(self.checks) | |
| def verify_all_faults_fixed( | |
| ground_truths: list[GroundTruth], network_manager | |
| ) -> VerificationResult: | |
| if not ground_truths: | |
| return VerificationResult(fixed=True, checks=["no faults injected"]) | |
| all_checks = [] | |
| all_failures = [] | |
| for gt in ground_truths: | |
| r = verify_fault_fixed(gt, network_manager) | |
| all_checks.extend(r.checks) | |
| all_failures.extend(r.failures) | |
| return VerificationResult( | |
| fixed=len(all_failures) == 0, | |
| checks=all_checks, | |
| failures=all_failures, | |
| ) | |
| def verify_fault_fixed( | |
| gt: GroundTruth, nm | |
| ) -> VerificationResult: | |
| fn = _VERIFIERS.get(gt.fault_type, _verify_unknown) | |
| try: | |
| return fn(gt, nm) | |
| except Exception as e: | |
| return VerificationResult( | |
| fixed=False, | |
| checks=[f"verify {gt.fault_type}"], | |
| failures=[f"verification error: {e}"], | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Helpers | |
| # --------------------------------------------------------------------------- | |
| def _get_router(nm, name): | |
| return nm.get_router(name) | |
| def _is_sim(): | |
| return os.environ.get("NETOPS_SIM") == "1" | |
| def _get_sim_router(nm, name): | |
| if hasattr(nm, '_engine') and nm._engine: | |
| return nm._engine.topo.routers.get(name) | |
| return None | |
| def _check_no_netem(nm, router_name, intf_name) -> tuple[bool, str]: | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| intf = sr.interfaces.get(intf_name) | |
| if intf and intf.has_netem: | |
| return False, f"{router_name}/{intf_name} still has netem" | |
| if intf and intf.has_tbf: | |
| return False, f"{router_name}/{intf_name} still has tbf" | |
| return True, f"{router_name}/{intf_name} netem clear" | |
| out = node.cmd(f"tc -s qdisc show dev {intf_name}") | |
| if "netem" in out or "tbf" in out: | |
| return False, f"{router_name}/{intf_name} still has netem/tbf" | |
| return True, f"{router_name}/{intf_name} netem clear" | |
| def _check_link_up(nm, router_name, intf_name) -> tuple[bool, str]: | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| intf = sr.interfaces.get(intf_name) | |
| if intf and not intf.up: | |
| return False, f"{router_name}/{intf_name} is DOWN" | |
| return True, f"{router_name}/{intf_name} is UP" | |
| try: | |
| ifaces = telemetry.read_interfaces(node) | |
| for iface in ifaces: | |
| if iface.get("ifname") == intf_name: | |
| state = iface.get("operstate", "UNKNOWN") | |
| if state.upper() != "UP": | |
| return False, f"{router_name}/{intf_name} operstate={state}" | |
| return True, f"{router_name}/{intf_name} is UP" | |
| except Exception: | |
| pass | |
| return True, f"{router_name}/{intf_name} state check skipped" | |
| def _check_bgp_sessions(nm, router_name) -> tuple[bool, str]: | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| try: | |
| bgp = telemetry.read_bgp_summary(node) | |
| peers = bgp.get("ipv4Unicast", {}).get("peers", {}) | |
| down = [pip for pip, pd in peers.items() if pd.get("state") != "Established"] | |
| if down: | |
| return False, f"{router_name} BGP sessions down: {down}" | |
| return True, f"{router_name} all BGP sessions Established" | |
| except Exception as e: | |
| return False, f"{router_name} BGP check failed: {e}" | |
| def _check_ospf_neighbors(nm, router_name) -> tuple[bool, str]: | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| try: | |
| ospf = telemetry.read_ospf_neighbors(node) | |
| neighbors = ospf.get("neighbors", ospf) | |
| for nid, ndata_wrap in neighbors.items(): | |
| entries = ndata_wrap if isinstance(ndata_wrap, list) else [ndata_wrap] | |
| for ndata in entries: | |
| state = ndata.get("nbrState", ndata.get("state", "")) | |
| if "/" in state: | |
| state = state.split("/")[0] | |
| if state != "Full": | |
| return False, f"{router_name} OSPF neighbor {nid} state={state}" | |
| return True, f"{router_name} all OSPF neighbors Full" | |
| except Exception as e: | |
| return False, f"{router_name} OSPF check failed: {e}" | |
| def _check_daemon_alive(nm, router_name, daemon) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| attr = f"{daemon.replace('d', '')}_daemon_alive" | |
| if hasattr(sr, attr) and not getattr(sr, attr): | |
| return False, f"{router_name} {daemon} not alive" | |
| return True, f"{router_name} {daemon} alive" | |
| if daemon == "bgpd": | |
| return _check_bgp_sessions(nm, router_name) | |
| elif daemon == "ospfd": | |
| return _check_ospf_neighbors(nm, router_name) | |
| return True, f"{router_name} {daemon} check skipped" | |
| def _check_no_iptables(nm, router_name) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| for intf in sr.interfaces.values(): | |
| if intf.iptables_output_drop: | |
| return False, f"{router_name}/{intf.name} has iptables OUTPUT DROP" | |
| if intf.iptables_input_rules: | |
| return False, f"{router_name} has iptables INPUT rules" | |
| if intf.iptables_forward_rules: | |
| return False, f"{router_name} has iptables FORWARD rules" | |
| return True, f"{router_name} no iptables rules" | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| out = node.cmd("iptables -L -n") | |
| for line in out.split("\n"): | |
| if "DROP" in line: | |
| return False, f"{router_name} has iptables DROP rules" | |
| return True, f"{router_name} no iptables DROP rules" | |
| def _check_no_static_route(nm, router_name, prefix) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr and prefix in sr.static_routes: | |
| return False, f"{router_name} still has static route {prefix}" | |
| return True, f"{router_name} no static route {prefix}" | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| out = node.cmd("ip route show type blackhole") | |
| if prefix in out: | |
| return False, f"{router_name} still has blackhole {prefix}" | |
| return True, f"{router_name} no blackhole {prefix}" | |
| def _check_no_bgp_network(nm, router_name, prefix) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr and prefix in sr.bgp_networks: | |
| return False, f"{router_name} still has BGP network {prefix}" | |
| return True, f"{router_name} no BGP network {prefix}" | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| out = node.cmd('vtysh -c "show running-config"') | |
| if f"network {prefix}" in out: | |
| return False, f"{router_name} still has BGP network {prefix}" | |
| return True, f"{router_name} no BGP network {prefix}" | |
| def _check_no_bgp_peer_policy(nm, router_name, peer_ip, policy_key) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| cfg = sr.bgp_peer_config.get(peer_ip, {}) | |
| if policy_key in cfg: | |
| return False, f"{router_name} peer {peer_ip} still has {policy_key}" | |
| return True, f"{router_name} peer {peer_ip} no {policy_key}" | |
| return True, f"{router_name} peer policy check skipped (Mininet)" | |
| def _check_ospf_cost(nm, router_name, intf_name, expected=10) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| cost = sr.ospf_costs.get(intf_name, 10) | |
| if cost != expected: | |
| return False, f"{router_name}/{intf_name} OSPF cost={cost} (expected {expected})" | |
| return True, f"{router_name}/{intf_name} OSPF cost={expected}" | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| try: | |
| ospf_if = telemetry.read_ospf_interface(node) | |
| ifaces = ospf_if.get("interfaces", ospf_if) | |
| for ifn, idata in ifaces.items(): | |
| if ifn == intf_name: | |
| cost = idata.get("cost", 10) | |
| if cost != expected: | |
| return False, f"{router_name}/{intf_name} cost={cost}" | |
| return True, f"{router_name}/{intf_name} cost={expected}" | |
| except Exception: | |
| pass | |
| return True, f"{router_name}/{intf_name} cost check skipped" | |
| def _check_ospf_timers(nm, router_name, intf_name) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| hello = sr.ospf_hello.get(intf_name, 10) | |
| dead = sr.ospf_dead.get(intf_name, 40) | |
| if hello != 10 or dead != 40: | |
| return False, f"{router_name}/{intf_name} timers hello={hello} dead={dead}" | |
| return True, f"{router_name}/{intf_name} timers normal" | |
| return True, f"{router_name}/{intf_name} timer check skipped" | |
| def _check_mtu(nm, router_name, intf_name, expected=1500) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| intf = sr.interfaces.get(intf_name) | |
| if intf and intf.mtu != expected: | |
| return False, f"{router_name}/{intf_name} MTU={intf.mtu}" | |
| return True, f"{router_name}/{intf_name} MTU={expected}" | |
| return True, f"{router_name}/{intf_name} MTU check skipped" | |
| def _check_ospf_area(nm, router_name, intf_name, expected="0.0.0.0") -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| area = sr.ospf_area.get(intf_name, "0.0.0.0") | |
| if area != expected: | |
| return False, f"{router_name}/{intf_name} area={area}" | |
| return True, f"{router_name}/{intf_name} area={expected}" | |
| return True, f"{router_name}/{intf_name} area check skipped" | |
| def _check_no_cpu_hog(nm, router_name) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr: | |
| if getattr(sr, 'cpu_exhausted', False) or getattr(sr, 'bg_cpu_hog', False): | |
| return False, f"{router_name} still has CPU hog" | |
| return True, f"{router_name} no CPU hog" | |
| node = _get_router(nm, router_name) | |
| if node is None: | |
| return False, f"{router_name} not found" | |
| out = node.cmd("pgrep -f 'gzip|dd if=/dev/urandom'") | |
| if out.strip(): | |
| return False, f"{router_name} CPU hog processes still running" | |
| return True, f"{router_name} no CPU hog" | |
| def _check_no_iperf(nm, router_name) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr and getattr(sr, 'bg_iperf', False): | |
| return False, f"{router_name} iperf still running" | |
| return True, f"{router_name} no iperf" | |
| return True, f"{router_name} iperf check skipped" | |
| def _check_no_flapping(nm, router_name, intf_name) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr and sr.bg_flapping.get(intf_name, False): | |
| return False, f"{router_name}/{intf_name} still flapping" | |
| return True, f"{router_name}/{intf_name} not flapping" | |
| return True, f"{router_name}/{intf_name} flap check skipped" | |
| def _check_no_redistribute_static(nm, router_name) -> tuple[bool, str]: | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router_name) | |
| if sr and sr.bgp_redistribute_static: | |
| return False, f"{router_name} still redistributes static" | |
| return True, f"{router_name} no redistribute static" | |
| return True, f"{router_name} redistribute check skipped" | |
| # --------------------------------------------------------------------------- | |
| # Per-fault verifiers | |
| # --------------------------------------------------------------------------- | |
| def _verify_unknown(gt, nm): | |
| return VerificationResult( | |
| fixed=False, | |
| checks=[f"no verifier for {gt.fault_type}"], | |
| failures=[f"unverifiable fault type: {gt.fault_type}"], | |
| ) | |
| def _verify_netem(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_netem(nm, d["router"], d["interface"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_link_failure(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for r, intf in [(d["router_a"], d["intf_a"]), (d["router_b"], d["intf_b"])]: | |
| ok, msg = _check_link_up(nm, r, intf) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_interface_flapping(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| ok, msg = _check_no_flapping(nm, d["router"], d["interface"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_link_up(nm, d["router"], d["interface"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_unidirectional(gt, nm): | |
| d = gt.details | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, d["router"]) | |
| if sr: | |
| intf = sr.interfaces.get(d["interface"]) | |
| if intf and intf.iptables_output_drop: | |
| return VerificationResult(fixed=False, | |
| checks=[f"{d['router']}/{d['interface']} OUTPUT DROP"], | |
| failures=[f"{d['router']}/{d['interface']} still has OUTPUT DROP"]) | |
| return VerificationResult(fixed=True, | |
| checks=[f"{d['router']}/{d['interface']} unidirectional check"]) | |
| def _verify_daemon_kill(gt, nm): | |
| d = gt.details | |
| daemon = d.get("daemon", "bgpd") | |
| checks, failures = [], [] | |
| ok, msg = _check_daemon_alive(nm, d["router"], daemon) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| if daemon == "bgpd": | |
| ok, msg = _check_bgp_sessions(nm, d["router"]) | |
| else: | |
| ok, msg = _check_ospf_neighbors(nm, d["router"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_ospf_cost(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_ospf_cost(nm, d["router"], d["interface"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_ospf_timer(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| ok, msg = _check_ospf_timers(nm, d["router"], d["interface"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_ospf_neighbors(nm, d["router"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_ospf_mtu(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_mtu(nm, d["router"], d["interface"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_ospf_area(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_ospf_area(nm, d["router"], d["interface"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bgp_route_withdrawal(gt, nm): | |
| d = gt.details | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, d["router"]) | |
| if sr: | |
| if not sr.bgp_redistribute_connected and not sr.bgp_redistribute_ospf: | |
| return VerificationResult(fixed=False, | |
| checks=[f"{d['router']} BGP redistribution"], | |
| failures=[f"{d['router']} no redistribute connected/ospf"]) | |
| ok, msg = _check_bgp_sessions(nm, d["router"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bgp_prefix_filter(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_bgp_peer_policy(nm, d["router"], d["peer_ip"], "prefix_list_in") | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bgp_session_flood(gt, nm): | |
| d = gt.details | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, d["router"]) | |
| if sr: | |
| flood_nets = [n for n in sr.bgp_networks if n.startswith("10.1")] | |
| flood_routes = [p for p in sr.static_routes if p.startswith("10.1")] | |
| if flood_nets or flood_routes: | |
| return VerificationResult(fixed=False, | |
| checks=[f"{d['router']} flood cleanup"], | |
| failures=[f"{d['router']} still has {len(flood_nets)} flood networks, {len(flood_routes)} flood routes"]) | |
| return VerificationResult(fixed=True, checks=[f"{d['router']} flood check"]) | |
| def _verify_bgp_route_leak(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_bgp_network(nm, d["router"], d["victim_prefix"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bgp_hijack(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for specific in d.get("specifics", []): | |
| ok, msg = _check_no_bgp_network(nm, d["router"], specific) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok2, msg2 = _check_no_static_route(nm, d["router"], specific) | |
| checks.append(msg2) | |
| if not ok2: | |
| failures.append(msg2) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_bgp_local_pref(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_bgp_peer_policy(nm, d["router"], d["peer_ip"], "local_pref_in") | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bgp_prepend(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_bgp_peer_policy(nm, d["router"], d["peer_ip"], "prepend_out") | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bgp_community(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_bgp_peer_policy(nm, d["router"], d["peer_ip"], "community_out") | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_selective_drop(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_iptables(nm, d["router"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_network_partition(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for r in [d["router_a"], d["router_b"]]: | |
| ok, msg = _check_no_iptables(nm, r) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_control_plane_isolation(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_iptables(nm, d["router"]) | |
| checks = [msg] | |
| failures = [msg] if not ok else [] | |
| ok2, msg2 = _check_bgp_sessions(nm, d["router"]) | |
| checks.append(msg2) | |
| if not ok2: | |
| failures.append(msg2) | |
| ok3, msg3 = _check_ospf_neighbors(nm, d["router"]) | |
| checks.append(msg3) | |
| if not ok3: | |
| failures.append(msg3) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_routing_loop(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for r in [d["router_a"], d["router_b"]]: | |
| ok, msg = _check_no_static_route(nm, r, d["target_prefix"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_bgp_blackhole(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| ok, msg = _check_no_static_route(nm, d["router"], d["prefix"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_no_redistribute_static(nm, d["router"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_silent_blackhole(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_static_route(nm, d["router"], d["summary_prefix"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_redistribution_loop(gt, nm): | |
| d = gt.details | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, d["router"]) | |
| if sr and sr.bgp_redistribute_static: | |
| return VerificationResult(fixed=False, | |
| checks=[f"{d['router']} redistribute"], | |
| failures=[f"{d['router']} still has redistribute static"]) | |
| return VerificationResult(fixed=True, checks=[f"{d['router']} redistribution check"]) | |
| def _verify_cpu_exhaustion(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_cpu_hog(nm, d["router"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_stale_routes(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_ospf_cost(nm, d["router"], d["interface"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_bandwidth_saturation(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_iperf(nm, d["router"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| # --- Novel faults --- | |
| def _verify_maintenance_drain(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for intf in d.get("drained_intfs", []): | |
| ok, msg = _check_ospf_cost(nm, d["router"], intf) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_daemon_alive(nm, d["router"], "ospfd") | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_config_typo_redundant(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for peer in d.get("filtered_peers", []): | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, d["router"]) | |
| if sr: | |
| for pip, cfg in sr.bgp_peer_config.items(): | |
| if "prefix_list_in" in cfg: | |
| msg = f"{d['router']} peer {pip} still has prefix filter" | |
| checks.append(msg) | |
| failures.append(msg) | |
| break | |
| else: | |
| msg = f"{d['router']} no prefix filters" | |
| checks.append(msg) | |
| else: | |
| checks.append(f"{d['router']} prefix filter check") | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_wrong_device(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_daemon_alive(nm, d["actual_target"], "bgpd") | |
| checks = [msg] | |
| failures = [msg] if not ok else [] | |
| ok2, msg2 = _check_bgp_sessions(nm, d["actual_target"]) | |
| checks.append(msg2) | |
| if not ok2: | |
| failures.append(msg2) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_micro_flapping(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_no_netem(nm, d["router"], d["interface"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_transient_loop(gt, nm): | |
| d = gt.details | |
| intf_name = None | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, d["router_a"]) | |
| if sr: | |
| for iname, cost in sr.ospf_costs.items(): | |
| if cost >= 1000: | |
| intf_name = iname | |
| break | |
| if intf_name: | |
| ok, msg = _check_ospf_cost(nm, d["router_a"], intf_name) | |
| else: | |
| nm_obj = _get_router(nm, d["router_a"]) | |
| if nm_obj: | |
| try: | |
| ospf_if = telemetry.read_ospf_interface(nm_obj) | |
| ifaces = ospf_if.get("interfaces", ospf_if) | |
| for ifn, idata in ifaces.items(): | |
| if idata.get("cost", 10) >= 1000: | |
| intf_name = ifn | |
| break | |
| except Exception: | |
| pass | |
| if intf_name: | |
| ok, msg = _check_ospf_cost(nm, d["router_a"], intf_name) | |
| else: | |
| ok, msg = True, f"{d['router_a']} no high-cost interfaces found" | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_correlated_outage(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| ok, msg = _check_daemon_alive(nm, d["router"], "ospfd") | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_daemon_alive(nm, d["router"], "bgpd") | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| for intf in d.get("downed_interfaces", []): | |
| ok, msg = _check_link_up(nm, d["router"], intf) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_dual_link_failure(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| for link_str in d.get("failed_links", []): | |
| parts = link_str.split("-") | |
| if len(parts) == 2: | |
| ra, rb = parts | |
| ok_a, msg_a = _check_bgp_sessions(nm, ra) | |
| checks.append(msg_a) | |
| if not ok_a: | |
| failures.append(msg_a) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_exact_prefix_hijack(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| ok, msg = _check_no_bgp_network(nm, d["hijacker"], d["victim_prefix"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_no_static_route(nm, d["hijacker"], d["victim_prefix"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_route_leak_business(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| ok, msg = _check_no_bgp_network(nm, d["leaker"], d["victim_prefix"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_silent_interception(gt, nm): | |
| d = gt.details | |
| checks, failures = [], [] | |
| subprefix = d.get("subprefix", "") | |
| if subprefix: | |
| ok, msg = _check_no_bgp_network(nm, d["interceptor"], subprefix) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_no_static_route(nm, d["interceptor"], subprefix) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| ok, msg = _check_no_redistribute_static(nm, d["interceptor"]) | |
| checks.append(msg) | |
| if not ok: | |
| failures.append(msg) | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| def _verify_ghost_fault(gt, nm): | |
| d = gt.details | |
| ok, msg = _check_bgp_sessions(nm, d["router"]) | |
| return VerificationResult(fixed=ok, checks=[msg], failures=[] if ok else [msg]) | |
| def _verify_red_herring_noisy(gt, nm): | |
| d = gt.details | |
| real_component = d.get("real_component", "") | |
| checks, failures = [], [] | |
| if "-" in real_component: | |
| parts = real_component.split("-") | |
| if len(parts) == 2: | |
| router, peer = parts | |
| if _is_sim(): | |
| sr = _get_sim_router(nm, router) | |
| if sr: | |
| for pip, cfg in sr.bgp_peer_config.items(): | |
| if "prefix_list_in" in cfg: | |
| msg = f"{router} peer {pip} still has prefix filter" | |
| checks.append(msg) | |
| failures.append(msg) | |
| break | |
| else: | |
| checks.append(f"{router} prefix filter removed") | |
| else: | |
| checks.append(f"{router} prefix filter check") | |
| return VerificationResult(fixed=len(failures) == 0, checks=checks, failures=failures) | |
| # Compound faults: these register sub-faults individually, | |
| # so verify_all_faults_fixed handles them by verifying each sub-fault. | |
| # The compound ground truth itself just needs to pass through. | |
| def _verify_compound_passthrough(gt, nm): | |
| return VerificationResult(fixed=True, checks=[f"{gt.fault_type} (sub-faults verified separately)"]) | |
| # --------------------------------------------------------------------------- | |
| # Dispatch table | |
| # --------------------------------------------------------------------------- | |
| _VERIFIERS = { | |
| # Physical / L2 | |
| "link_failure": _verify_link_failure, | |
| "latency_spike": _verify_netem, | |
| "packet_loss": _verify_netem, | |
| "packet_corruption": _verify_netem, | |
| "packet_reordering": _verify_netem, | |
| "packet_duplication": _verify_netem, | |
| "interface_flapping": _verify_interface_flapping, | |
| "unidirectional_failure": _verify_unidirectional, | |
| # QoS | |
| "bandwidth_throttle": _verify_netem, | |
| "bandwidth_saturation": _verify_bandwidth_saturation, | |
| "jitter": _verify_netem, | |
| # OSPF | |
| "ospf_cost_change": _verify_ospf_cost, | |
| "ospf_timer_mismatch": _verify_ospf_timer, | |
| "ospf_mtu_mismatch": _verify_ospf_mtu, | |
| "ospf_area_mismatch": _verify_ospf_area, | |
| "ospf_daemon_kill": _verify_daemon_kill, | |
| # BGP control | |
| "bgp_session_kill": _verify_daemon_kill, | |
| "bgp_route_withdrawal": _verify_bgp_route_withdrawal, | |
| "bgp_session_flood": _verify_bgp_session_flood, | |
| "bgp_prefix_filter": _verify_bgp_prefix_filter, | |
| # BGP policy | |
| "bgp_route_leak": _verify_bgp_route_leak, | |
| "bgp_hijack": _verify_bgp_hijack, | |
| "bgp_local_pref": _verify_bgp_local_pref, | |
| "bgp_prepend": _verify_bgp_prepend, | |
| "bgp_community": _verify_bgp_community, | |
| # Firewall | |
| "selective_drop_bgp": _verify_selective_drop, | |
| "network_partition": _verify_network_partition, | |
| "control_plane_isolation": _verify_control_plane_isolation, | |
| # Routing logic | |
| "routing_loop": _verify_routing_loop, | |
| "bgp_blackhole": _verify_bgp_blackhole, | |
| "silent_blackhole_aggregate": _verify_silent_blackhole, | |
| "redistribution_loop": _verify_redistribution_loop, | |
| # System | |
| "cpu_exhaustion": _verify_cpu_exhaustion, | |
| "stale_routes_after_crash": _verify_stale_routes, | |
| # Compound (sub-faults verified individually) | |
| "multi_fault_easy": _verify_compound_passthrough, | |
| "multi_fault_hard": _verify_compound_passthrough, | |
| "cascading_failure": _verify_compound_passthrough, | |
| "red_herring_cpu": _verify_compound_passthrough, | |
| "red_herring_flap": _verify_compound_passthrough, | |
| "triple_fault": _verify_compound_passthrough, | |
| "cross_layer_fault": _verify_compound_passthrough, | |
| "cascading_reveal": _verify_compound_passthrough, | |
| # Novel | |
| "maintenance_drain_failure": _verify_maintenance_drain, | |
| "config_typo_redundant": _verify_config_typo_redundant, | |
| "wrong_device": _verify_wrong_device, | |
| "micro_flapping": _verify_micro_flapping, | |
| "transient_loop": _verify_transient_loop, | |
| "correlated_outage": _verify_correlated_outage, | |
| "dual_link_failure": _verify_dual_link_failure, | |
| "exact_prefix_hijack": _verify_exact_prefix_hijack, | |
| "route_leak_business": _verify_route_leak_business, | |
| "silent_interception": _verify_silent_interception, | |
| "ghost_fault": _verify_ghost_fault, | |
| "red_herring_noisy": _verify_red_herring_noisy, | |
| } | |