Spaces:
Running
Running
Download tests/test_faults.py from brand0nwalls/NetOps: direct link, hf CLI and curl.
- Browser
- Download file 5.22 kB
-
https://huggingface.co/spaces/brand0nwalls/NetOps/resolve/main/tests/test_faults.py
- Command line
-
hf download hf://spaces/brand0nwalls/NetOps/tests/test_faults.py
-
curl -L -o test_faults.py https://huggingface.co/spaces/brand0nwalls/NetOps/resolve/main/tests/test_faults.py
5.22 kB
| """Test fault injection: inject, verify effect, undo, verify recovery.""" | |
| import os | |
| import sys | |
| import time | |
| import json | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from server.network import NetworkManager | |
| from server.fault_injector import FaultInjector | |
| from server import telemetry | |
| def test_faults(): | |
| nm = NetworkManager() | |
| passed = 0 | |
| failed = 0 | |
| print("[0] Starting network...") | |
| nm.start() | |
| fi = FaultInjector(nm) | |
| # ========== LINK FAILURE ========== | |
| print("\n[1] LINK FAILURE: r1-r4 (eBGP link)") | |
| print(" Injecting...") | |
| gt = fi.inject_link_failure("r1", "r4") | |
| assert gt.fault_type == "link_failure" | |
| assert gt.component == "r1-r4" | |
| time.sleep(2) | |
| r1 = nm.get_router("r1") | |
| intfs = telemetry.read_interfaces(r1) | |
| r1_r4_intf = gt.details["intf_a"] | |
| link_down = False | |
| for i in intfs: | |
| if i.get("ifname") == r1_r4_intf: | |
| flags = i.get("flags", []) | |
| if "UP" not in flags and "LOWER_UP" not in flags: | |
| link_down = True | |
| if link_down: | |
| print(f" PASS: {r1_r4_intf} is DOWN") | |
| passed += 1 | |
| else: | |
| print(f" FAIL: {r1_r4_intf} still appears UP") | |
| failed += 1 | |
| print(" Undoing...") | |
| fi.undo_all() | |
| time.sleep(3) | |
| intfs = telemetry.read_interfaces(r1) | |
| link_up = False | |
| for i in intfs: | |
| if i.get("ifname") == r1_r4_intf: | |
| flags = i.get("flags", []) | |
| if "UP" in flags or "LOWER_UP" in flags: | |
| link_up = True | |
| if link_up: | |
| print(f" PASS: {r1_r4_intf} restored to UP") | |
| passed += 1 | |
| else: | |
| print(f" FAIL: {r1_r4_intf} not restored") | |
| failed += 1 | |
| # Wait for BGP to re-establish after undo | |
| print(" Waiting for BGP re-convergence...") | |
| conv = nm.wait_for_bgp(timeout=60) | |
| if conv > 0: | |
| print(f" BGP re-converged in {conv:.1f}s") | |
| else: | |
| print(" WARNING: BGP did not re-converge (continuing anyway)") | |
| # ========== BGP SESSION KILL ========== | |
| print("\n[2] BGP SESSION KILL: r5") | |
| print(" Injecting...") | |
| gt = fi.inject_bgp_session_kill("r5") | |
| assert gt.fault_type == "bgp_session_kill" | |
| assert gt.component == "r5" | |
| time.sleep(3) | |
| r5 = nm.get_router("r5") | |
| bgpd_pid = FaultInjector._find_daemon_pid(r5, "bgpd") | |
| if bgpd_pid is None: | |
| print(" PASS: bgpd killed on r5 (no PID in namespace)") | |
| passed += 1 | |
| else: | |
| print(f" FAIL: bgpd still running on r5 (PID {bgpd_pid})") | |
| failed += 1 | |
| print(" Undoing...") | |
| fi.undo_all() | |
| time.sleep(5) | |
| bgpd_pid = FaultInjector._find_daemon_pid(r5, "bgpd") | |
| if bgpd_pid is not None: | |
| print(f" PASS: bgpd restarted on r5 (PID {bgpd_pid})") | |
| passed += 1 | |
| else: | |
| print(" FAIL: bgpd not restarted on r5") | |
| failed += 1 | |
| # Wait for BGP to re-establish | |
| print(" Waiting for BGP re-convergence...") | |
| conv = nm.wait_for_bgp(timeout=90) | |
| if conv > 0: | |
| print(f" BGP re-converged in {conv:.1f}s") | |
| else: | |
| print(" WARNING: BGP did not fully re-converge") | |
| # ========== LATENCY SPIKE ========== | |
| print("\n[3] LATENCY SPIKE: r1->r2 200ms") | |
| print(" Baseline ping...") | |
| baseline = nm.ping("h1", "h2", count=2) | |
| print(f" Baseline loss: {baseline['loss_pct']}%") | |
| print(" Injecting 200ms latency on r1->r2...") | |
| gt = fi.inject_latency("r1", "r2", delay_ms=200) | |
| assert gt.fault_type == "latency_spike" | |
| time.sleep(1) | |
| r1 = nm.get_router("r1") | |
| tc = telemetry.read_queue_stats(r1, gt.details["interface"]) | |
| has_netem = any(q.get("type") == "netem" for q in tc.get("qdiscs", [])) | |
| if has_netem: | |
| print(" PASS: netem qdisc installed") | |
| passed += 1 | |
| else: | |
| print(f" FAIL: no netem qdisc found. qdiscs: {tc.get('qdiscs', [])}") | |
| failed += 1 | |
| print(" Undoing...") | |
| fi.undo_all() | |
| time.sleep(1) | |
| tc = telemetry.read_queue_stats(r1, gt.details["interface"]) | |
| has_netem = any(q.get("type") == "netem" for q in tc.get("qdiscs", [])) | |
| if not has_netem: | |
| print(" PASS: netem removed") | |
| passed += 1 | |
| else: | |
| print(" FAIL: netem still present") | |
| failed += 1 | |
| # ========== GROUND TRUTH TRACKING ========== | |
| print("\n[4] Ground truth tracking...") | |
| fi.inject_link_failure("r2", "r3") | |
| fi.inject_latency("r7", "r8", delay_ms=100) | |
| gts = fi.ground_truths | |
| if len(gts) == 2: | |
| print(f" PASS: {len(gts)} ground truths tracked") | |
| passed += 1 | |
| else: | |
| print(f" FAIL: expected 2, got {len(gts)}") | |
| failed += 1 | |
| fi.undo_all() | |
| if len(fi.active_faults) == 0: | |
| print(" PASS: all faults cleared after undo_all") | |
| passed += 1 | |
| else: | |
| print(f" FAIL: {len(fi.active_faults)} faults still active") | |
| failed += 1 | |
| print("\n[cleanup] Stopping network...") | |
| nm.stop() | |
| print(f"\n{'='*40}") | |
| print(f"RESULTS: {passed} passed, {failed} failed") | |
| print(f"{'='*40}") | |
| return failed == 0 | |
| if __name__ == "__main__": | |
| success = test_faults() | |
| sys.exit(0 if success else 1) | |