from __future__ import annotations import uuid from typing import Dict, Any, List from models import Action, ActionType from tasks.base import BaseTask, InternalState, StepOutput, semantic_match # Based on real Cloudflare Global Outage (July 2, 2019) # Post-mortem: https://blog.cloudflare.com/details-of-the-cloudflare-outage-on-july-2-2019/ # A single WAF rule with a catastrophically backtracking regex ('^(a+)+$') # was fast-tracked to production without staging testing. CPU exhaustion # across all edge nodes caused ~27 minutes of global downtime. INCIDENT_TIME = "2019-07-02T14:02:15Z" DEPENDENCIES = [ {"service": "waf-service", "calls": [], "called_by": ["edge-node-us", "edge-node-eu"]}, {"service": "edge-node-us", "calls": ["waf-service", "origin-api"], "called_by": []}, {"service": "edge-node-eu", "calls": ["waf-service", "origin-api"], "called_by": []}, {"service": "origin-api", "calls": [], "called_by": ["edge-node-us", "edge-node-eu"]}, {"service": "waf-rule-manager", "calls": ["waf-service"], "called_by": []}, ] WAF_SERVICE_LOGS = [ "[14:02:15] INFO WAF rule deployment: rule-set v2.4.1 applied (47 rules)", "[14:02:16] INFO New rule SIG-2019-07-02: pattern='^(a+)+$' applied to URI matching", "[14:02:17] WARN CPU spike detected: 45% → 78% immediately after rule deployment", "[14:02:20] ERROR CPU at 98% — regex engine backtracking on URI: /api/search?q=aaaaaaaaab", "[14:02:21] ERROR Rule SIG-2019-07-02 triggering catastrophic backtracking", "[14:02:22] FATAL CPU 100% — all worker threads blocked in regex evaluation", "[14:02:23] FATAL Dropping 100% of incoming requests — WAF unresponsive", ] EDGE_US_LOGS = [ "[14:02:18] WARN WAF processing time: avg 4200ms (was 12ms pre-deployment)", "[14:02:20] ERROR Requests timing out waiting for WAF verdict", "[14:02:22] ERROR Error rate: 87% — WAF saturation", "[14:02:24] CRIT All requests being dropped — circuit breaker open", ] EDGE_EU_LOGS = [ "[14:02:18] WARN WAF processing time: avg 4700ms (was 11ms pre-deployment)", "[14:02:20] ERROR Requests timing out waiting for WAF verdict (EU edge)", "[14:02:21] ERROR Error rate: 91% — WAF saturation", "[14:02:23] CRIT All requests being dropped — circuit breaker open", ] WAF_RULE_MANAGER_LOGS = [ "[14:02:15] INFO Deployed rule-set v2.4.1 to production (fast-track, no staging test)", "[14:02:15] INFO New rule SIG-2019-07-02 added: ReDoS risk not evaluated", "[14:02:17] WARN CPU spike on waf-service detected post-deploy", "[14:02:20] INFO Previous rule-set v2.4.0 available for rollback", ] ORIGIN_API_LOGS = [ "[14:02:15] INFO Healthy — serving 4200 req/s at avg 28ms", "[14:02:23] WARN Incoming request rate dropped: 4200 → 0 req/s", "[14:02:24] INFO No requests reaching origin — edge layer saturated by WAF", ] class WafTask(BaseTask): def initialize(self) -> InternalState: logs = { "waf-service": WAF_SERVICE_LOGS[:], "edge-node-us": EDGE_US_LOGS[:], "edge-node-eu": EDGE_EU_LOGS[:], "waf-rule-manager": WAF_RULE_MANAGER_LOGS[:], "origin-api": ORIGIN_API_LOGS[:], } services: Dict[str, dict] = { "waf-service": { "name": "waf-service", "status": "down", "cpu_percent": 100.0, "memory_percent": 55.0, "error_rate": 100.0, "latency_p99_ms": 0.0, "replicas_running": 0, "replicas_desired": 12, "current_version": "rule-set-v2.4.1", "last_deployed": "2019-07-02T14:02:15Z", "minutes_degraded": 0, "sla_breach": False, }, "edge-node-us": { "name": "edge-node-us", "status": "degraded", "cpu_percent": 97.0, "memory_percent": 70.0, "error_rate": 87.0, "latency_p99_ms": 4200.0, "replicas_running": 8, "replicas_desired": 8, "current_version": "v5.1.0", "last_deployed": "2019-06-28T10:00:00Z", "minutes_degraded": 0, "sla_breach": False, }, "edge-node-eu": { "name": "edge-node-eu", "status": "degraded", "cpu_percent": 98.0, "memory_percent": 72.0, "error_rate": 91.0, "latency_p99_ms": 4700.0, "replicas_running": 6, "replicas_desired": 6, "current_version": "v5.1.0", "last_deployed": "2019-06-28T10:00:00Z", "minutes_degraded": 0, "sla_breach": False, }, "origin-api": { "name": "origin-api", "status": "healthy", "cpu_percent": 12.0, "memory_percent": 28.0, "error_rate": 0.0, "latency_p99_ms": 28.0, "replicas_running": 10, "replicas_desired": 10, "current_version": "v4.0.2", "last_deployed": "2019-07-01T06:00:00Z", "minutes_degraded": 0, "sla_breach": False, }, "waf-rule-manager": { "name": "waf-rule-manager", "status": "healthy", "cpu_percent": 8.0, "memory_percent": 15.0, "error_rate": 0.0, "latency_p99_ms": 35.0, "replicas_running": 2, "replicas_desired": 2, "current_version": "rule-set-v2.4.1", "last_deployed": "2019-07-02T14:02:15Z", "minutes_degraded": 0, "sla_breach": False, }, } alerts = [ { "id": "A001", "severity": "critical", "service": "waf-service", "message": "CPU 100% sustained — all request processing halted (regex backtracking)", "timestamp": "2019-07-02T14:02:22Z", "acknowledged": False, }, { "id": "A002", "severity": "critical", "service": "edge-node-us", "message": "87% request error rate — WAF unresponsive", "timestamp": "2019-07-02T14:02:22Z", "acknowledged": False, }, { "id": "A003", "severity": "critical", "service": "edge-node-eu", "message": "91% request error rate — WAF unresponsive", "timestamp": "2019-07-02T14:02:23Z", "acknowledged": False, }, { "id": "A004", "severity": "warning", "service": "waf-rule-manager", "message": "Rule deployment v2.4.1 correlates with CPU spike — rollback available", "timestamp": "2019-07-02T14:02:17Z", "acknowledged": False, }, ] state = InternalState( episode_id=str(uuid.uuid4()), task_id="waf", step=0, max_steps=18, services=services, alerts=alerts, logs=logs, action_history=[], total_reward=0.0, incident_resolved=False, ground_truth_root_cause="malformed_waf_regex_rule_cpu_exhaustion_excessive_backtracking", ground_truth_fix="rollback waf-rule-manager to previous rule set", incident_start_time=INCIDENT_TIME, healthy_services=["origin-api", "waf-rule-manager"], service_dependencies=DEPENDENCIES, ) return state def step(self, state: InternalState, action: Action) -> StepOutput: state.step += 1 state._apply_sla_degradation() at = action.action_type svc = action.service or "" reward = 0.0 done = False info: Dict[str, Any] = {} result_text, error_text = self._apply_action_to_logs(state, action) # --- Information-gathering rewards --- gather_map = { ("read_logs", "waf-service"): ("rl_waf", 0.10), ("search_logs", "waf-service"): ("rl_waf", 0.10), ("read_logs", "waf-rule-manager"): ("rl_wafmgr", 0.10), ("search_logs", "waf-rule-manager"): ("rl_wafmgr", 0.10), ("read_metrics", "waf-service"): ("rm_waf", 0.05), } k = (at.value, svc) if k in gather_map: tag, r = gather_map[k] if tag not in state.rewards_given: reward += r state.rewards_given.add(tag) if at == ActionType.READ_RUNBOOK: if "runbook" not in state.rewards_given: reward += 0.05 state.rewards_given.add("runbook") # --- Diagnose --- if at == ActionType.DIAGNOSE: rc = action.root_cause or "" waf_keywords = ["regex", "waf", "rule", "backtrack", "cpu", "pattern", "redos", "sig-2019", "exhaustion"] if semantic_match(rc, waf_keywords, threshold=1): if "diagnose_correct" not in state.rewards_given: reward += 0.20 state.rewards_given.add("diagnose_correct") result_text = f"Diagnosis recorded: {rc}" # --- Correct fix: Rollback waf-rule-manager --- if at == ActionType.ROLLBACK and svc == "waf-rule-manager": blind_penalty = self._penalty_blind_remediation(state, action, "fix_rollback") reward += blind_penalty if "fix_rollback" not in state.rewards_given: reward += 0.35 state.rewards_given.add("fix_rollback") # Bonus for specifying the correct rollback version version = (action.version or "").lower() if "v2.4.0" in version or "previous" in version: if "bonus_version" not in state.rewards_given: reward += 0.15 state.rewards_given.add("bonus_version") # Apply the fix state.services["waf-service"]["status"] = "healthy" state.services["waf-service"]["cpu_percent"] = 38.0 state.services["waf-service"]["error_rate"] = 0.0 state.services["waf-service"]["latency_p99_ms"] = 12.0 state.services["waf-service"]["replicas_running"] = 12 state.services["edge-node-us"]["status"] = "healthy" state.services["edge-node-us"]["error_rate"] = 0.0 state.services["edge-node-eu"]["status"] = "healthy" state.services["edge-node-eu"]["error_rate"] = 0.0 state.services["waf-rule-manager"]["current_version"] = "rule-set-v2.4.0" state.alerts = [] state.incident_resolved = True done = True info["resolution"] = "incident_resolved" result_text = ( "WAF rule-set rolled back to v2.4.0. " "SIG-2019-07-02 (catastrophic backtracking regex) removed. " "CPU normalized: 100% → 38%. All edge nodes recovering. " "Request processing resumed globally." ) # --- Penalty: Restarting waf-service won't fix it — rule reloads on start --- if at == ActionType.RESTART_SERVICE and svc == "waf-service": reward -= 0.15 result_text = ( "waf-service restarted — but the bad WAF rule is still in rule-set v2.4.1. " "Restarting reloads the same rule configuration. CPU immediately hit 100% again. " "Restart does not fix a bad regex rule — you must rollback the rule-set." ) # Penalty: scaling up edge nodes with same bad rule = same problem if at == ActionType.RESTART_SERVICE and svc in ("edge-node-us", "edge-node-eu"): reward -= 0.10 result_text = f"{svc}: New nodes joined but inherited the same WAF rule-set v2.4.1. CPU still 100%." # Penalty: collateral damage if at == ActionType.RESTART_SERVICE and svc in state.healthy_services: reward -= 0.10 error_text = f"Collateral damage: {svc} was healthy. Unnecessary restart." if at == ActionType.NOOP and state.step > 3: reward -= 0.04 if at in (ActionType.BLOCK_IP_RANGE, ActionType.CREATE_INDEX, ActionType.FAILOVER): reward -= 0.10 error_text = f"Action {at.value} is not applicable to a WAF CPU exhaustion incident." state.total_reward = self._clamp(state.total_reward + reward) if state.step >= state.max_steps and not done: done = True info["reason"] = "max_steps_reached" obs = state._build_observation(last_action_result=result_text, last_action_error=error_text) state.action_history.append({"step": state.step, "action": action.model_dump(), "reward": round(reward, 4)}) return StepOutput(next_state=state, reward=round(reward, 4), done=done, info=info)