Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| import uuid | |
| from typing import Dict, Any, List | |
| from models import Action, ActionType | |
| from tasks.base import BaseTask, InternalState, StepOutput, semantic_match | |
| # Based on real GitHub DNS Outage (January 2016) | |
| # Post-mortem: https://github.blog/news-insights/the-library/dns-outage-post-mortem/ | |
| # A Puppet manifest bug restarted only the authoritative nameserver (not the caching one). | |
| # A follow-up re-deploy corrupted the zone file further because the zone rebuild | |
| # script depended on DNS itself — which was already broken. | |
| INCIDENT_TIME = "2016-01-28T09:14:22Z" | |
| DEPENDENCIES = [ | |
| {"service": "api-gateway", "calls": ["dns-caching"], "called_by": []}, | |
| {"service": "dns-caching", "calls": ["dns-authoritative"], "called_by": ["api-gateway"]}, | |
| {"service": "dns-authoritative", "calls": [], "called_by": ["dns-caching", "deploy-service"]}, | |
| {"service": "deploy-service", "calls": ["dns-authoritative"], "called_by": []}, | |
| {"service": "puppet-master", "calls": [], "called_by": []}, | |
| ] | |
| DNS_AUTH_LOGS = [ | |
| "[09:14:22] INFO Puppet run started: nameserver-ip-update manifest", | |
| "[09:14:45] INFO Restarting authoritative nameserver... done", | |
| "[09:14:46] WARN Caching nameserver NOT restarted (not in manifest scope)", | |
| "[09:14:47] INFO Zone transfer initiated to caching nameserver", | |
| "[09:14:52] ERROR Zone transfer failed: caching nameserver has no new config", | |
| "[09:15:01] ERROR deploy-service called zone-rebuild API during incident", | |
| "[09:15:03] ERROR zone-rebuild API made DNS query → got NXDOMAIN → built corrupt zone", | |
| "[09:15:04] FATAL Zone file now corrupt: 847 records returning NXDOMAIN", | |
| ] | |
| DNS_CACHING_LOGS = [ | |
| "[09:14:22] INFO Serving cached zone: last updated 09:00:00", | |
| "[09:14:47] WARN Zone transfer attempted from authoritative — rejected (version mismatch)", | |
| "[09:14:50] WARN Continuing to serve stale zone (30+ minutes old)", | |
| "[09:14:55] ERROR api-gateway queries failing: NXDOMAIN for payment.internal", | |
| "[09:14:57] ERROR api-gateway queries failing: NXDOMAIN for orders.internal", | |
| "[09:15:10] CRIT 847 DNS queries/min returning NXDOMAIN — zone corrupted", | |
| ] | |
| DEPLOY_SERVICE_LOGS = [ | |
| "[09:15:00] INFO Attempting re-deploy to fix DNS issue", | |
| "[09:15:02] INFO Calling zone-rebuild API at zone-manager.internal", | |
| "[09:15:03] ERROR DNS resolution failed for zone-manager.internal: NXDOMAIN", | |
| "[09:15:03] WARN Zone rebuild called with null zone data — wrote empty zone", | |
| "[09:15:04] CRIT Zone rebuild made situation worse — do not re-deploy", | |
| ] | |
| PUPPET_LOGS = [ | |
| "[09:14:22] INFO Puppet agent run triggered: nameserver-ip-update", | |
| "[09:14:45] INFO Resource File[/etc/named.conf] updated", | |
| "[09:14:45] INFO Service[named-authoritative] restarted", | |
| "[09:14:46] INFO Puppet run completed successfully", | |
| "[09:14:46] WARN Note: named-caching.service not in manifest — skipped", | |
| ] | |
| API_GATEWAY_LOGS = [ | |
| "[09:14:53] WARN DNS lookup failed: payment.internal → NXDOMAIN", | |
| "[09:14:54] WARN DNS lookup failed: orders.internal → NXDOMAIN", | |
| "[09:14:55] ERROR 94% of service-discovery lookups returning NXDOMAIN", | |
| "[09:14:56] CRIT Upstream services unreachable — DNS failure cascade", | |
| ] | |
| class DnsTask(BaseTask): | |
| def initialize(self) -> InternalState: | |
| logs = { | |
| "dns-authoritative": DNS_AUTH_LOGS[:], | |
| "dns-caching": DNS_CACHING_LOGS[:], | |
| "deploy-service": DEPLOY_SERVICE_LOGS[:], | |
| "puppet-master": PUPPET_LOGS[:], | |
| "api-gateway": API_GATEWAY_LOGS[:], | |
| } | |
| services: Dict[str, dict] = { | |
| "dns-authoritative": { | |
| "name": "dns-authoritative", "status": "down", | |
| "cpu_percent": 12.0, "memory_percent": 88.0, | |
| "error_rate": 847.0, "latency_p99_ms": 0.0, | |
| "replicas_running": 0, "replicas_desired": 1, | |
| "current_version": "bind-9.11.3-puppet-r142", | |
| "last_deployed": "2016-01-28T09:14:45Z", | |
| "minutes_degraded": 0, "sla_breach": False, | |
| }, | |
| "dns-caching": { | |
| "name": "dns-caching", "status": "degraded", | |
| "cpu_percent": 45.0, "memory_percent": 62.0, | |
| "error_rate": 94.0, "latency_p99_ms": 5200.0, | |
| "replicas_running": 1, "replicas_desired": 1, | |
| "current_version": "unbound-1.5.8", | |
| "last_deployed": "2016-01-10T00:00:00Z", | |
| "minutes_degraded": 0, "sla_breach": False, | |
| }, | |
| "puppet-master": { | |
| "name": "puppet-master", "status": "healthy", | |
| "cpu_percent": 22.0, "memory_percent": 35.0, | |
| "error_rate": 0.0, "latency_p99_ms": 120.0, | |
| "replicas_running": 1, "replicas_desired": 1, | |
| "current_version": "puppet-4.3.2-r142", | |
| "last_deployed": "2016-01-28T09:14:22Z", | |
| "minutes_degraded": 0, "sla_breach": False, | |
| }, | |
| "api-gateway": { | |
| "name": "api-gateway", "status": "down", | |
| "cpu_percent": 18.0, "memory_percent": 41.0, | |
| "error_rate": 94.0, "latency_p99_ms": 30000.0, | |
| "replicas_running": 0, "replicas_desired": 5, | |
| "current_version": "v2.4.0", | |
| "last_deployed": "2016-01-20T08:00:00Z", | |
| "minutes_degraded": 0, "sla_breach": False, | |
| }, | |
| "deploy-service": { | |
| "name": "deploy-service", "status": "degraded", | |
| "cpu_percent": 33.0, "memory_percent": 28.0, | |
| "error_rate": 12.0, "latency_p99_ms": 8000.0, | |
| "replicas_running": 1, "replicas_desired": 1, | |
| "current_version": "v1.8.0", | |
| "last_deployed": "2016-01-28T09:15:00Z", | |
| "minutes_degraded": 0, "sla_breach": False, | |
| }, | |
| } | |
| alerts = [ | |
| { | |
| "id": "A001", "severity": "critical", "service": "api-gateway", | |
| "message": "94% DNS queries returning NXDOMAIN — services unreachable", | |
| "timestamp": "2016-01-28T09:14:55Z", "acknowledged": False, | |
| }, | |
| { | |
| "id": "A002", "severity": "critical", "service": "dns-authoritative", | |
| "message": "Zone file corrupted — 847 records returning NXDOMAIN", | |
| "timestamp": "2016-01-28T09:15:04Z", "acknowledged": False, | |
| }, | |
| { | |
| "id": "A003", "severity": "warning", "service": "dns-caching", | |
| "message": "Serving stale zone — zone transfer rejected (version mismatch)", | |
| "timestamp": "2016-01-28T09:14:47Z", "acknowledged": False, | |
| }, | |
| { | |
| "id": "A004", "severity": "info", "service": "puppet-master", | |
| "message": "Puppet run completed successfully (manifest bug already applied)", | |
| "timestamp": "2016-01-28T09:14:46Z", "acknowledged": False, | |
| }, | |
| ] | |
| state = InternalState( | |
| episode_id=str(uuid.uuid4()), | |
| task_id="dns", | |
| step=0, | |
| max_steps=20, | |
| services=services, | |
| alerts=alerts, | |
| logs=logs, | |
| action_history=[], | |
| total_reward=0.0, | |
| incident_resolved=False, | |
| ground_truth_root_cause="puppet_manifest_bug_restarted_wrong_nameserver_corrupt_dns_zone", | |
| ground_truth_fix="restart dns-caching nameserver AND rollback puppet manifest", | |
| incident_start_time=INCIDENT_TIME, | |
| healthy_services=["puppet-master"], | |
| service_dependencies=DEPENDENCIES, | |
| ) | |
| return state | |
| def step(self, state: InternalState, action: Action) -> StepOutput: | |
| state.step += 1 | |
| state._apply_sla_degradation() | |
| at = action.action_type | |
| svc = action.service or "" | |
| reward = 0.0 | |
| done = False | |
| info: Dict[str, Any] = {} | |
| result_text, error_text = self._apply_action_to_logs(state, action) | |
| # --- Information-gathering rewards --- | |
| gather_map = { | |
| ("read_logs", "dns-authoritative"): ("rl_dns_auth", 0.10), | |
| ("search_logs", "dns-authoritative"): ("rl_dns_auth", 0.10), | |
| ("read_logs", "dns-caching"): ("rl_dns_cache", 0.10), | |
| ("search_logs", "dns-caching"): ("rl_dns_cache", 0.10), | |
| ("read_logs", "deploy-service"): ("rl_deploy", 0.05), | |
| ("search_logs", "deploy-service"): ("rl_deploy", 0.05), | |
| } | |
| k = (at.value, svc) | |
| if k in gather_map: | |
| tag, r = gather_map[k] | |
| if tag not in state.rewards_given: | |
| reward += r | |
| state.rewards_given.add(tag) | |
| if at == ActionType.READ_RUNBOOK: | |
| if "runbook" not in state.rewards_given: | |
| reward += 0.05 | |
| state.rewards_given.add("runbook") | |
| # --- Diagnose --- | |
| if at == ActionType.DIAGNOSE: | |
| rc = action.root_cause or "" | |
| dns_keywords = ["puppet", "manifest", "nameserver", "dns", "zone", "nxdomain", "caching", "corrupt"] | |
| if semantic_match(rc, dns_keywords, threshold=1): | |
| if "diagnose_correct" not in state.rewards_given: | |
| reward += 0.20 | |
| state.rewards_given.add("diagnose_correct") | |
| result_text = f"Diagnosis recorded: {rc}" | |
| # --- Fix 1: Restart dns-caching (primary fix — restarts with new config) --- | |
| if at == ActionType.RESTART_SERVICE and svc == "dns-caching": | |
| blind_penalty = self._penalty_blind_remediation(state, action, "fix_cache_restart") | |
| reward += blind_penalty | |
| if "fix_cache_restart" not in state.rewards_given: | |
| reward += 0.25 | |
| state.rewards_given.add("fix_cache_restart") | |
| state.services["dns-caching"]["status"] = "healthy" | |
| state.services["dns-caching"]["error_rate"] = 0.0 | |
| state.services["dns-caching"]["latency_p99_ms"] = 5.0 | |
| state.services["api-gateway"]["error_rate"] = 0.0 | |
| state.services["api-gateway"]["status"] = "healthy" | |
| state.services["api-gateway"]["replicas_running"] = 5 | |
| state.alerts = [a for a in state.alerts if a["id"] not in ("A002", "A003")] | |
| result_text = ( | |
| "dns-caching restarted. Loaded fresh config from authoritative nameserver. " | |
| "Zone transfer successful. DNS resolution restored. " | |
| "api-gateway recovering — NXDOMAIN rate dropping." | |
| ) | |
| if "fix_puppet_rollback" in state.rewards_given: | |
| state.incident_resolved = True | |
| done = True | |
| info["resolution"] = "incident_resolved" | |
| # --- Fix 2: Rollback puppet-master (prevents recurrence) --- | |
| if at == ActionType.ROLLBACK and svc == "puppet-master": | |
| if "fix_puppet_rollback" not in state.rewards_given: | |
| reward += 0.15 | |
| state.rewards_given.add("fix_puppet_rollback") | |
| state.services["puppet-master"]["current_version"] = "puppet-4.3.2-r141" | |
| result_text = ( | |
| "Puppet manifest rolled back to r141. " | |
| "The nameserver-ip-update manifest has been reverted. " | |
| "Future Puppet runs will restart both authoritative AND caching nameservers." | |
| ) | |
| if "fix_cache_restart" in state.rewards_given: | |
| state.incident_resolved = True | |
| done = True | |
| info["resolution"] = "incident_resolved" | |
| # --- Penalty: Running deploy-service makes it worse --- | |
| if at == ActionType.RESTART_SERVICE and svc == "deploy-service": | |
| reward -= 0.20 | |
| result_text = ( | |
| "CRITICAL: Re-running deploy-service triggered another zone rebuild attempt. " | |
| "zone-rebuild API made DNS query → got NXDOMAIN → zone corruption worsened. " | |
| "DO NOT re-deploy during a DNS incident — the deploy script depends on DNS." | |
| ) | |
| # Penalty: restarting healthy puppet-master without diagnosis | |
| if at == ActionType.RESTART_SERVICE and svc == "puppet-master": | |
| reward -= 0.05 | |
| result_text = "puppet-master restarted. The manifest bug is still present — use rollback instead." | |
| # Penalty: collateral damage on healthy services | |
| if at == ActionType.RESTART_SERVICE and svc in state.healthy_services: | |
| reward -= 0.10 | |
| error_text = f"Collateral damage: {svc} was healthy. Unnecessary restart." | |
| if at == ActionType.NOOP and state.step > 4: | |
| reward -= 0.04 | |
| if at in (ActionType.BLOCK_IP_RANGE, ActionType.CREATE_INDEX, ActionType.FAILOVER): | |
| reward -= 0.10 | |
| error_text = f"Action {at.value} is not applicable to a DNS configuration incident." | |
| state.total_reward = self._clamp(state.total_reward + reward) | |
| if state.step >= state.max_steps and not done: | |
| done = True | |
| info["reason"] = "max_steps_reached" | |
| obs = state._build_observation(last_action_result=result_text, last_action_error=error_text) | |
| state.action_history.append({"step": state.step, "action": action.model_dump(), "reward": round(reward, 4)}) | |
| return StepOutput(next_state=state, reward=round(reward, 4), done=done, info=info) | |