Spaces:
Running
Running
Download inference.py from brand0nwalls/NetOps: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/spaces/brand0nwalls/NetOps/resolve/main/inference.py
- Command line
-
hf download hf://spaces/brand0nwalls/NetOps/inference.py
-
curl -L -o inference.py https://huggingface.co/spaces/brand0nwalls/NetOps/resolve/main/inference.py
15.5 kB
| """Inference script for NetOps environment. | |
| Runs an LLM agent through network fault diagnosis and remediation episodes. | |
| Uses OpenAI-compatible API (works with Ollama, HF router, vLLM, etc.). | |
| Stdout format follows OpenEnv validation spec: | |
| [START] task=<task> env=netops model=<model> | |
| [STEP] step=<n> action=<action> reward=<0.00> done=<true|false> error=<msg|null> | |
| [END] task=<task> success=<true|false> steps=<n> score=<0.000> rewards=<r1,r2,...> | |
| """ | |
| import json | |
| import os | |
| import sys | |
| import textwrap | |
| from typing import List, Optional | |
| import requests | |
| from openai import OpenAI | |
| API_KEY = os.environ.get("API_KEY") or os.environ.get("HF_TOKEN") or "ollama" | |
| API_BASE_URL = os.environ.get("API_BASE_URL") or "http://localhost:11434/v1" | |
| MODEL_NAME = os.environ.get("MODEL_NAME") or "gemma4:31b" | |
| ENV_URL = os.environ.get("ENV_URL", "") | |
| BENCHMARK = "netops" | |
| TASK_FILTER = os.environ.get("TASK_FILTER", "") | |
| MAX_STEPS = 20 | |
| TEMPERATURE = 0.3 | |
| MAX_TOKENS = 512 | |
| USE_LOCAL = os.environ.get("USE_LOCAL", "1") == "1" | |
| TASKS = [ | |
| # Easy β L1/L2 faults (6) | |
| ("link_failure", "easy", {"fault_type": "link_failure"}), | |
| ("latency_spike", "easy", {"fault_type": "latency_spike"}), | |
| ("packet_loss", "easy", {"fault_type": "packet_loss"}), | |
| ("packet_corruption", "easy", {"fault_type": "packet_corruption"}), | |
| ("packet_duplication", "easy", {"fault_type": "packet_duplication"}), | |
| ("packet_reordering", "easy", {"fault_type": "packet_reordering"}), | |
| # Medium β OSPF + basic BGP + QoS (12) | |
| ("interface_flapping", "medium", {"fault_type": "interface_flapping"}), | |
| ("ospf_cost_change", "medium", {"fault_type": "ospf_cost_change"}), | |
| ("ospf_timer_mismatch", "medium", {"fault_type": "ospf_timer_mismatch"}), | |
| ("ospf_mtu_mismatch", "medium", {"fault_type": "ospf_mtu_mismatch"}), | |
| ("ospf_area_mismatch", "medium", {"fault_type": "ospf_area_mismatch"}), | |
| ("ospf_daemon_kill", "medium", {"fault_type": "ospf_daemon_kill"}), | |
| ("bgp_session_kill", "medium", {"fault_type": "bgp_session_kill"}), | |
| ("bgp_route_withdrawal", "medium", {"fault_type": "bgp_route_withdrawal"}), | |
| ("selective_drop_bgp", "medium", {"fault_type": "selective_drop_bgp"}), | |
| ("bandwidth_throttle", "medium", {"fault_type": "bandwidth_throttle"}), | |
| ("bandwidth_saturation", "medium", {"fault_type": "bandwidth_saturation"}), | |
| ("unidirectional_failure", "medium", {"fault_type": "unidirectional_failure"}), | |
| # Hard β BGP policy + routing logic + system + compound (16) | |
| ("bgp_blackhole", "hard", {"fault_type": "bgp_blackhole"}), | |
| ("bgp_hijack", "hard", {"fault_type": "bgp_hijack"}), | |
| ("bgp_local_pref", "hard", {"fault_type": "bgp_local_pref"}), | |
| ("bgp_prepend", "hard", {"fault_type": "bgp_prepend"}), | |
| ("bgp_community", "hard", {"fault_type": "bgp_community"}), | |
| ("bgp_route_leak", "hard", {"fault_type": "bgp_route_leak"}), | |
| ("bgp_prefix_filter", "hard", {"fault_type": "bgp_prefix_filter"}), | |
| ("bgp_session_flood", "hard", {"fault_type": "bgp_session_flood"}), | |
| ("routing_loop", "hard", {"fault_type": "routing_loop"}), | |
| ("network_partition", "hard", {"fault_type": "network_partition"}), | |
| ("control_plane_isolation", "hard", {"fault_type": "control_plane_isolation"}), | |
| ("cpu_exhaustion", "hard", {"fault_type": "cpu_exhaustion"}), | |
| ("redistribution_loop", "hard", {"fault_type": "redistribution_loop"}), | |
| ("stale_routes_after_crash", "hard", {"fault_type": "stale_routes_after_crash"}), | |
| ("jitter", "hard", {"fault_type": "jitter"}), | |
| ("silent_blackhole_aggregate", "hard", {"fault_type": "silent_blackhole_aggregate"}), | |
| ("multi_fault_easy", "hard", {"fault_type": "multi_fault_easy"}), | |
| ("multi_fault_hard", "hard", {"fault_type": "multi_fault_hard"}), | |
| ("cascading_failure", "hard", {"fault_type": "cascading_failure"}), | |
| # Expert β red herrings, multi-layer, cascading reveal (5) | |
| ("red_herring_cpu", "expert", {"fault_type": "red_herring_cpu"}), | |
| ("red_herring_flap", "expert", {"fault_type": "red_herring_flap"}), | |
| ("triple_fault", "expert", {"fault_type": "triple_fault"}), | |
| ("cross_layer_fault", "expert", {"fault_type": "cross_layer_fault"}), | |
| ("cascading_reveal", "expert", {"fault_type": "cascading_reveal"}), | |
| # Novel β research-grounded faults (12) | |
| ("maintenance_drain_failure", "expert", {"fault_type": "maintenance_drain_failure"}), | |
| ("config_typo_redundant", "expert", {"fault_type": "config_typo_redundant"}), | |
| ("wrong_device", "hard", {"fault_type": "wrong_device"}), | |
| ("micro_flapping", "hard", {"fault_type": "micro_flapping"}), | |
| ("transient_loop", "expert", {"fault_type": "transient_loop"}), | |
| ("correlated_outage", "hard", {"fault_type": "correlated_outage"}), | |
| ("dual_link_failure", "hard", {"fault_type": "dual_link_failure"}), | |
| ("exact_prefix_hijack", "hard", {"fault_type": "exact_prefix_hijack"}), | |
| ("route_leak_business", "expert", {"fault_type": "route_leak_business"}), | |
| ("silent_interception", "expert", {"fault_type": "silent_interception"}), | |
| ("ghost_fault", "expert", {"fault_type": "ghost_fault"}), | |
| ("red_herring_noisy", "expert", {"fault_type": "red_herring_noisy"}), | |
| ] | |
| SYSTEM_PROMPT = textwrap.dedent(""" | |
| You are an expert network operations engineer diagnosing and fixing faults in a | |
| multi-AS network running OSPF within each AS and eBGP between ASes. | |
| The network has multiple routers and hosts across several autonomous systems. | |
| Each turn you must respond with EXACTLY one JSON action: | |
| {"command": "<cmd>", "args": {<args>}} | |
| INVESTIGATION commands (gather information): | |
| query_routing(router) β show BGP summary, OSPF neighbors, routes for a router | |
| query_topology() β show all routers' BGP/OSPF/interface status | |
| query_device(router) β show interface stats, counters, tc rules for a router | |
| query_traffic() β show traffic stats and flow summaries | |
| query_events() β show recent network events and alerts | |
| ping(src, dst) β ping between hosts or routers (e.g. src="h1", dst="h5") | |
| traceroute(src, dst) β trace path between hosts or routers | |
| DIAGNOSIS (after investigation): | |
| diagnose(fault_type, component, explanation) | |
| component: "r1-r4" for a link/pair, "r1" for a single router | |
| REMEDIATION (pick the right fix for the diagnosed issue): | |
| restore_link(router1, router2) β bring a downed link back up | |
| restart_daemon(router, daemon) β restart a daemon (ospfd, bgpd) | |
| remove_tc(router, interface) β remove netem rules (fixes latency/loss/corruption/reorder/jitter/duplication) | |
| remove_iptables(router) β flush firewall rules | |
| restore_mtu(router, interface) β restore default MTU | |
| restore_ospf_timers(router) β restore default OSPF hello/dead timers | |
| restore_ospf_area(router) β restore correct OSPF area assignment | |
| adjust_cost(router, interface, cost) β set OSPF interface cost | |
| remove_route(router, prefix) β remove a static/blackhole route | |
| remove_bgp_config(router) β remove injected BGP policy changes | |
| kill_traffic(router) β stop traffic generators (iperf, etc.) | |
| remove_cpu_hog(router) β stop CPU-exhausting processes | |
| stop_flapping(router, interface) β kill interface flapping loop and bring link up | |
| remove_bgp_flood(router, num_prefixes) β remove flood-injected BGP prefixes and routes | |
| restore_drain(router) β restore all OSPF costs to default (undo maintenance drain) | |
| restore_all_intfs(router) β bring all interfaces up on a router | |
| noop() β do nothing | |
| EPISODE CONTROL: | |
| confirm_fix() β declare the fault fixed and end the episode | |
| Strategy: | |
| 1. Investigate: use 2-3 DIFFERENT query types to understand the fault | |
| 2. Diagnose: identify the fault type and affected component | |
| 3. Remediate: apply 1-2 targeted fix actions matching your diagnosis | |
| 4. Verify: query again to confirm the fix worked | |
| 5. End: call confirm_fix() once you believe the issue is resolved | |
| Never repeat the exact same action twice in a row. | |
| Reply with ONLY the JSON action, no explanation. | |
| """).strip() | |
| def log_start(task, env, model): | |
| print(f"[START] task={task} env={env} model={model}", flush=True) | |
| def log_step(step, action, reward, done, error): | |
| error_val = error if error else "null" | |
| print(f"[STEP] step={step} action={action[:200]} reward={reward:.2f} done={str(done).lower()} error={error_val}", flush=True) | |
| def log_end(task, success, steps, score, rewards): | |
| rewards_str = ",".join(f"{r:.2f}" for r in rewards) | |
| print(f"[END] task={task} success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True) | |
| _local_env = None | |
| def _get_local_env(): | |
| global _local_env | |
| if _local_env is None: | |
| os.environ["NETOPS_SIM"] = "1" | |
| from server.environment import NetOpsEnvironment | |
| _local_env = NetOpsEnvironment() | |
| return _local_env | |
| def env_reset(difficulty="easy", fault_type=None): | |
| if USE_LOCAL: | |
| from models import NetOpsAction | |
| env = _get_local_env() | |
| obs = env.reset(difficulty=difficulty, fault_type=fault_type) | |
| return {"observation": obs.model_dump(), "done": obs.done, "reward": obs.reward} | |
| payload = {"difficulty": difficulty} | |
| if fault_type: | |
| payload["fault_type"] = fault_type | |
| resp = requests.post(f"{ENV_URL}/reset", json=payload, timeout=60) | |
| resp.raise_for_status() | |
| return resp.json() | |
| def env_step(action_dict): | |
| if USE_LOCAL: | |
| from models import NetOpsAction | |
| env = _get_local_env() | |
| action = NetOpsAction(**action_dict) | |
| obs = env.step(action) | |
| return {"observation": obs.model_dump(), "done": obs.done, "reward": obs.reward} | |
| resp = requests.post(f"{ENV_URL}/step", json={"action": action_dict}, timeout=60) | |
| resp.raise_for_status() | |
| return resp.json() | |
| def get_llm_action(client, history, alert, observation): | |
| user_msg = f"ALERT: {alert}\n\nLATEST OBSERVATION:\n{observation}\n\nRespond with ONE JSON action." | |
| messages = [{"role": "system", "content": SYSTEM_PROMPT}] | |
| for h in history[-6:]: | |
| messages.append({"role": "assistant", "content": json.dumps(h["action"])}) | |
| messages.append({"role": "user", "content": f"Result:\n{h['observation']}"}) | |
| messages.append({"role": "user", "content": user_msg}) | |
| try: | |
| completion = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=messages, | |
| temperature=TEMPERATURE, | |
| max_tokens=MAX_TOKENS, | |
| ) | |
| text = (completion.choices[0].message.content or "").strip() | |
| _last_raw_response[0] = text | |
| start = text.find("{") | |
| end = text.rfind("}") + 1 | |
| if start >= 0 and end > start: | |
| return json.loads(text[start:end]) | |
| return {"command": "query_topology", "args": {}} | |
| except Exception as e: | |
| print(f"[DEBUG] LLM error: {e}", flush=True) | |
| _last_raw_response[0] = f"ERROR: {e}" | |
| return {"command": "query_topology", "args": {}} | |
| _last_raw_response = [""] | |
| def run_episode(client, task_id, difficulty, fault_type=None): | |
| rewards = [] | |
| steps = 0 | |
| score = 0.001 | |
| success = False | |
| episode_history = [] | |
| log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME) | |
| try: | |
| reset_resp = env_reset(difficulty, fault_type=fault_type) | |
| obs_data = reset_resp.get("observation", {}) | |
| alert = obs_data.get("alert", "") | |
| observation = obs_data.get("observation", "") | |
| done = reset_resp.get("done", False) | |
| history = [] | |
| print(f"[DEBUG] Alert: {alert}", flush=True) | |
| for step in range(1, MAX_STEPS + 1): | |
| if done: | |
| break | |
| action = get_llm_action(client, history, alert, observation) | |
| action_str = json.dumps(action) | |
| step_resp = env_step(action) | |
| obs_data = step_resp.get("observation", {}) | |
| reward = step_resp.get("reward") or 0.0 | |
| done = step_resp.get("done", False) | |
| observation = obs_data.get("observation", "") | |
| rewards.append(reward) | |
| steps = step | |
| history.append({ | |
| "action": action, | |
| "observation": observation[:500], | |
| "reward": reward, | |
| }) | |
| fault_fixed = "Fault fixed: True" in observation if done else None | |
| episode_history.append({ | |
| "step": step, | |
| "action": action, | |
| "model_raw_response": _last_raw_response[0], | |
| "observation": observation, | |
| "reward": reward, | |
| "done": done, | |
| "fault_fixed": fault_fixed, | |
| }) | |
| log_step(step=step, action=action_str, reward=reward, done=done, error=None) | |
| if done: | |
| break | |
| total_reward = sum(rewards) | |
| score = max(0.001, min(0.999, (total_reward + 3.05) / 6.55)) | |
| success = episode_history[-1].get("fault_fixed", False) if episode_history else False | |
| except Exception as e: | |
| print(f"[DEBUG] Episode error: {e}", flush=True) | |
| score = 0.001 | |
| finally: | |
| log_end(task=task_id, success=success, steps=steps, score=score, rewards=rewards) | |
| episode_data = { | |
| "task_id": task_id, | |
| "difficulty": difficulty, | |
| "alert": alert if 'alert' in dir() else "", | |
| "score": round(score, 4), | |
| "success": success, | |
| "steps": steps, | |
| "total_reward": round(total_reward, 4) if 'total_reward' in dir() else 0, | |
| "rewards": [round(r, 4) for r in rewards], | |
| "history": episode_history, | |
| } | |
| return score, episode_data | |
| def main(): | |
| client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY) | |
| print(f"[DEBUG] Model: {MODEL_NAME}", flush=True) | |
| print(f"[DEBUG] Env: {ENV_URL}", flush=True) | |
| print(f"[DEBUG] API: {API_BASE_URL}", flush=True) | |
| all_results = [] | |
| task_list = TASKS | |
| if TASK_FILTER: | |
| allowed = set(TASK_FILTER.split(",")) | |
| task_list = [(t, d, k) for t, d, k in TASKS if t in allowed] | |
| print(f"[DEBUG] Task filter: {sorted(allowed)} β {len(task_list)} tasks", flush=True) | |
| for task_id, difficulty, kwargs in task_list: | |
| fault_type = kwargs.get("fault_type") | |
| score, episode_data = run_episode(client, task_id, difficulty, fault_type=fault_type) | |
| all_results.append(episode_data) | |
| avg = sum(r["score"] for r in all_results) / len(all_results) if all_results else 0 | |
| print(f"\n[SUMMARY] {len(all_results)} tasks, avg score={avg:.3f}", flush=True) | |
| model_slug = MODEL_NAME.replace("/", "_").replace(":", "_") | |
| os.makedirs("results", exist_ok=True) | |
| results_path = f"results/{model_slug}_results.json" | |
| output = { | |
| "model": MODEL_NAME, | |
| "api_base_url": API_BASE_URL, | |
| "env": BENCHMARK, | |
| "avg_score": round(avg, 4), | |
| "episodes": all_results, | |
| } | |
| with open(results_path, "w") as f: | |
| json.dump(output, f, indent=2) | |
| print(f"[DEBUG] Results saved to {results_path}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |