Spaces:
Sleeping
Sleeping
File size: 13,851 Bytes
03cf367 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 | from __future__ import annotations
import uuid
from typing import Dict, Any, List
from models import Action, ActionType
from tasks.base import BaseTask, InternalState, StepOutput, semantic_match
# Based on real GitHub DNS Outage (January 2016)
# Post-mortem: https://github.blog/news-insights/the-library/dns-outage-post-mortem/
# A Puppet manifest bug restarted only the authoritative nameserver (not the caching one).
# A follow-up re-deploy corrupted the zone file further because the zone rebuild
# script depended on DNS itself — which was already broken.
INCIDENT_TIME = "2016-01-28T09:14:22Z"
DEPENDENCIES = [
{"service": "api-gateway", "calls": ["dns-caching"], "called_by": []},
{"service": "dns-caching", "calls": ["dns-authoritative"], "called_by": ["api-gateway"]},
{"service": "dns-authoritative", "calls": [], "called_by": ["dns-caching", "deploy-service"]},
{"service": "deploy-service", "calls": ["dns-authoritative"], "called_by": []},
{"service": "puppet-master", "calls": [], "called_by": []},
]
DNS_AUTH_LOGS = [
"[09:14:22] INFO Puppet run started: nameserver-ip-update manifest",
"[09:14:45] INFO Restarting authoritative nameserver... done",
"[09:14:46] WARN Caching nameserver NOT restarted (not in manifest scope)",
"[09:14:47] INFO Zone transfer initiated to caching nameserver",
"[09:14:52] ERROR Zone transfer failed: caching nameserver has no new config",
"[09:15:01] ERROR deploy-service called zone-rebuild API during incident",
"[09:15:03] ERROR zone-rebuild API made DNS query → got NXDOMAIN → built corrupt zone",
"[09:15:04] FATAL Zone file now corrupt: 847 records returning NXDOMAIN",
]
DNS_CACHING_LOGS = [
"[09:14:22] INFO Serving cached zone: last updated 09:00:00",
"[09:14:47] WARN Zone transfer attempted from authoritative — rejected (version mismatch)",
"[09:14:50] WARN Continuing to serve stale zone (30+ minutes old)",
"[09:14:55] ERROR api-gateway queries failing: NXDOMAIN for payment.internal",
"[09:14:57] ERROR api-gateway queries failing: NXDOMAIN for orders.internal",
"[09:15:10] CRIT 847 DNS queries/min returning NXDOMAIN — zone corrupted",
]
DEPLOY_SERVICE_LOGS = [
"[09:15:00] INFO Attempting re-deploy to fix DNS issue",
"[09:15:02] INFO Calling zone-rebuild API at zone-manager.internal",
"[09:15:03] ERROR DNS resolution failed for zone-manager.internal: NXDOMAIN",
"[09:15:03] WARN Zone rebuild called with null zone data — wrote empty zone",
"[09:15:04] CRIT Zone rebuild made situation worse — do not re-deploy",
]
PUPPET_LOGS = [
"[09:14:22] INFO Puppet agent run triggered: nameserver-ip-update",
"[09:14:45] INFO Resource File[/etc/named.conf] updated",
"[09:14:45] INFO Service[named-authoritative] restarted",
"[09:14:46] INFO Puppet run completed successfully",
"[09:14:46] WARN Note: named-caching.service not in manifest — skipped",
]
API_GATEWAY_LOGS = [
"[09:14:53] WARN DNS lookup failed: payment.internal → NXDOMAIN",
"[09:14:54] WARN DNS lookup failed: orders.internal → NXDOMAIN",
"[09:14:55] ERROR 94% of service-discovery lookups returning NXDOMAIN",
"[09:14:56] CRIT Upstream services unreachable — DNS failure cascade",
]
class DnsTask(BaseTask):
def initialize(self) -> InternalState:
logs = {
"dns-authoritative": DNS_AUTH_LOGS[:],
"dns-caching": DNS_CACHING_LOGS[:],
"deploy-service": DEPLOY_SERVICE_LOGS[:],
"puppet-master": PUPPET_LOGS[:],
"api-gateway": API_GATEWAY_LOGS[:],
}
services: Dict[str, dict] = {
"dns-authoritative": {
"name": "dns-authoritative", "status": "down",
"cpu_percent": 12.0, "memory_percent": 88.0,
"error_rate": 847.0, "latency_p99_ms": 0.0,
"replicas_running": 0, "replicas_desired": 1,
"current_version": "bind-9.11.3-puppet-r142",
"last_deployed": "2016-01-28T09:14:45Z",
"minutes_degraded": 0, "sla_breach": False,
},
"dns-caching": {
"name": "dns-caching", "status": "degraded",
"cpu_percent": 45.0, "memory_percent": 62.0,
"error_rate": 94.0, "latency_p99_ms": 5200.0,
"replicas_running": 1, "replicas_desired": 1,
"current_version": "unbound-1.5.8",
"last_deployed": "2016-01-10T00:00:00Z",
"minutes_degraded": 0, "sla_breach": False,
},
"puppet-master": {
"name": "puppet-master", "status": "healthy",
"cpu_percent": 22.0, "memory_percent": 35.0,
"error_rate": 0.0, "latency_p99_ms": 120.0,
"replicas_running": 1, "replicas_desired": 1,
"current_version": "puppet-4.3.2-r142",
"last_deployed": "2016-01-28T09:14:22Z",
"minutes_degraded": 0, "sla_breach": False,
},
"api-gateway": {
"name": "api-gateway", "status": "down",
"cpu_percent": 18.0, "memory_percent": 41.0,
"error_rate": 94.0, "latency_p99_ms": 30000.0,
"replicas_running": 0, "replicas_desired": 5,
"current_version": "v2.4.0",
"last_deployed": "2016-01-20T08:00:00Z",
"minutes_degraded": 0, "sla_breach": False,
},
"deploy-service": {
"name": "deploy-service", "status": "degraded",
"cpu_percent": 33.0, "memory_percent": 28.0,
"error_rate": 12.0, "latency_p99_ms": 8000.0,
"replicas_running": 1, "replicas_desired": 1,
"current_version": "v1.8.0",
"last_deployed": "2016-01-28T09:15:00Z",
"minutes_degraded": 0, "sla_breach": False,
},
}
alerts = [
{
"id": "A001", "severity": "critical", "service": "api-gateway",
"message": "94% DNS queries returning NXDOMAIN — services unreachable",
"timestamp": "2016-01-28T09:14:55Z", "acknowledged": False,
},
{
"id": "A002", "severity": "critical", "service": "dns-authoritative",
"message": "Zone file corrupted — 847 records returning NXDOMAIN",
"timestamp": "2016-01-28T09:15:04Z", "acknowledged": False,
},
{
"id": "A003", "severity": "warning", "service": "dns-caching",
"message": "Serving stale zone — zone transfer rejected (version mismatch)",
"timestamp": "2016-01-28T09:14:47Z", "acknowledged": False,
},
{
"id": "A004", "severity": "info", "service": "puppet-master",
"message": "Puppet run completed successfully (manifest bug already applied)",
"timestamp": "2016-01-28T09:14:46Z", "acknowledged": False,
},
]
state = InternalState(
episode_id=str(uuid.uuid4()),
task_id="dns",
step=0,
max_steps=20,
services=services,
alerts=alerts,
logs=logs,
action_history=[],
total_reward=0.0,
incident_resolved=False,
ground_truth_root_cause="puppet_manifest_bug_restarted_wrong_nameserver_corrupt_dns_zone",
ground_truth_fix="restart dns-caching nameserver AND rollback puppet manifest",
incident_start_time=INCIDENT_TIME,
healthy_services=["puppet-master"],
service_dependencies=DEPENDENCIES,
)
return state
def step(self, state: InternalState, action: Action) -> StepOutput:
state.step += 1
state._apply_sla_degradation()
at = action.action_type
svc = action.service or ""
reward = 0.0
done = False
info: Dict[str, Any] = {}
result_text, error_text = self._apply_action_to_logs(state, action)
# --- Information-gathering rewards ---
gather_map = {
("read_logs", "dns-authoritative"): ("rl_dns_auth", 0.10),
("search_logs", "dns-authoritative"): ("rl_dns_auth", 0.10),
("read_logs", "dns-caching"): ("rl_dns_cache", 0.10),
("search_logs", "dns-caching"): ("rl_dns_cache", 0.10),
("read_logs", "deploy-service"): ("rl_deploy", 0.05),
("search_logs", "deploy-service"): ("rl_deploy", 0.05),
}
k = (at.value, svc)
if k in gather_map:
tag, r = gather_map[k]
if tag not in state.rewards_given:
reward += r
state.rewards_given.add(tag)
if at == ActionType.READ_RUNBOOK:
if "runbook" not in state.rewards_given:
reward += 0.05
state.rewards_given.add("runbook")
# --- Diagnose ---
if at == ActionType.DIAGNOSE:
rc = action.root_cause or ""
dns_keywords = ["puppet", "manifest", "nameserver", "dns", "zone", "nxdomain", "caching", "corrupt"]
if semantic_match(rc, dns_keywords, threshold=1):
if "diagnose_correct" not in state.rewards_given:
reward += 0.20
state.rewards_given.add("diagnose_correct")
result_text = f"Diagnosis recorded: {rc}"
# --- Fix 1: Restart dns-caching (primary fix — restarts with new config) ---
if at == ActionType.RESTART_SERVICE and svc == "dns-caching":
blind_penalty = self._penalty_blind_remediation(state, action, "fix_cache_restart")
reward += blind_penalty
if "fix_cache_restart" not in state.rewards_given:
reward += 0.25
state.rewards_given.add("fix_cache_restart")
state.services["dns-caching"]["status"] = "healthy"
state.services["dns-caching"]["error_rate"] = 0.0
state.services["dns-caching"]["latency_p99_ms"] = 5.0
state.services["api-gateway"]["error_rate"] = 0.0
state.services["api-gateway"]["status"] = "healthy"
state.services["api-gateway"]["replicas_running"] = 5
state.alerts = [a for a in state.alerts if a["id"] not in ("A002", "A003")]
result_text = (
"dns-caching restarted. Loaded fresh config from authoritative nameserver. "
"Zone transfer successful. DNS resolution restored. "
"api-gateway recovering — NXDOMAIN rate dropping."
)
if "fix_puppet_rollback" in state.rewards_given:
state.incident_resolved = True
done = True
info["resolution"] = "incident_resolved"
# --- Fix 2: Rollback puppet-master (prevents recurrence) ---
if at == ActionType.ROLLBACK and svc == "puppet-master":
if "fix_puppet_rollback" not in state.rewards_given:
reward += 0.15
state.rewards_given.add("fix_puppet_rollback")
state.services["puppet-master"]["current_version"] = "puppet-4.3.2-r141"
result_text = (
"Puppet manifest rolled back to r141. "
"The nameserver-ip-update manifest has been reverted. "
"Future Puppet runs will restart both authoritative AND caching nameservers."
)
if "fix_cache_restart" in state.rewards_given:
state.incident_resolved = True
done = True
info["resolution"] = "incident_resolved"
# --- Penalty: Running deploy-service makes it worse ---
if at == ActionType.RESTART_SERVICE and svc == "deploy-service":
reward -= 0.20
result_text = (
"CRITICAL: Re-running deploy-service triggered another zone rebuild attempt. "
"zone-rebuild API made DNS query → got NXDOMAIN → zone corruption worsened. "
"DO NOT re-deploy during a DNS incident — the deploy script depends on DNS."
)
# Penalty: restarting healthy puppet-master without diagnosis
if at == ActionType.RESTART_SERVICE and svc == "puppet-master":
reward -= 0.05
result_text = "puppet-master restarted. The manifest bug is still present — use rollback instead."
# Penalty: collateral damage on healthy services
if at == ActionType.RESTART_SERVICE and svc in state.healthy_services:
reward -= 0.10
error_text = f"Collateral damage: {svc} was healthy. Unnecessary restart."
if at == ActionType.NOOP and state.step > 4:
reward -= 0.04
if at in (ActionType.BLOCK_IP_RANGE, ActionType.CREATE_INDEX, ActionType.FAILOVER):
reward -= 0.10
error_text = f"Action {at.value} is not applicable to a DNS configuration incident."
state.total_reward = self._clamp(state.total_reward + reward)
if state.step >= state.max_steps and not done:
done = True
info["reason"] = "max_steps_reached"
obs = state._build_observation(last_action_result=result_text, last_action_error=error_text)
state.action_history.append({"step": state.step, "action": action.model_dump(), "reward": round(reward, 4)})
return StepOutput(next_state=state, reward=round(reward, 4), done=done, info=info)
|