Publish static Async Task Reliability Lab demo
Browse files- README.md +11 -4
- engine.py +83 -0
- index.html +57 -18
- test_engine.py +32 -0
README.md
CHANGED
|
@@ -1,10 +1,17 @@
|
|
| 1 |
---
|
| 2 |
title: Async Task Reliability Lab
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
|
|
|
| 7 |
pinned: false
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: Async Task Reliability Lab
|
| 3 |
+
emoji: 🔁
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
sdk: static
|
| 7 |
+
app_file: index.html
|
| 8 |
pinned: false
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# Async Task Reliability Lab
|
| 12 |
+
|
| 13 |
+
An interactive, browser-only backend reliability lab. Inject a worker failure and compare naive retries, idempotency keys, and a transactional outbox. The event timeline and ledger make the ambiguous “committed, but the response was lost” case visible.
|
| 14 |
+
|
| 15 |
+
The hosted simulator is static HTML/JavaScript and runs without a paid Space runtime. The repository also includes a small Python reference engine (`engine.py`) and unit tests (`test_engine.py`); run them locally with `python -m unittest -v`.
|
| 16 |
+
|
| 17 |
+
All records and side effects are synthetic and in memory. This demo does not charge money, call a real service, or claim production guarantees. The outbox example illustrates the atomic-write boundary; a production system still needs durable storage, delivery workers, consumer deduplication, observability, and recovery.
|
engine.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, asdict
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
@dataclass
|
| 5 |
+
class Event:
|
| 6 |
+
time: str
|
| 7 |
+
kind: str
|
| 8 |
+
detail: str
|
| 9 |
+
tone: str = "info"
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class Result:
|
| 13 |
+
mode: str
|
| 14 |
+
scenario: str
|
| 15 |
+
attempts: int
|
| 16 |
+
side_effects: int
|
| 17 |
+
status: str
|
| 18 |
+
ledger: list
|
| 19 |
+
events: list
|
| 20 |
+
explanation: str
|
| 21 |
+
|
| 22 |
+
SCENARIOS = {
|
| 23 |
+
"Crash after commit (response lost)": "commit_lost",
|
| 24 |
+
"Worker crashes before commit": "before_commit",
|
| 25 |
+
"Transient delivery failure": "transient",
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
MODES = {
|
| 29 |
+
"Naive retry": "naive",
|
| 30 |
+
"Idempotency key": "idempotent",
|
| 31 |
+
"Transactional outbox": "outbox",
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
def simulate(mode: str, scenario: str, max_retries: int = 3) -> Result:
|
| 35 |
+
mode_key = MODES.get(mode, mode)
|
| 36 |
+
scenario_key = SCENARIOS.get(scenario, scenario)
|
| 37 |
+
events, ledger = [], []
|
| 38 |
+
attempts = side_effects = 0
|
| 39 |
+
key = "order-7f2a"
|
| 40 |
+
events.append(Event("T+00ms", "request", f"POST /charge · idempotency-key: {key}").__dict__)
|
| 41 |
+
if mode_key == "outbox":
|
| 42 |
+
events.append(Event("T+02ms", "database", "Order + outbox event committed in one transaction.", "good").__dict__)
|
| 43 |
+
|
| 44 |
+
for attempt in range(1, max_retries + 2):
|
| 45 |
+
attempts = attempt
|
| 46 |
+
events.append(Event(f"T+{attempt*20:03d}ms", "worker", f"Attempt {attempt}: handling {key}.").__dict__)
|
| 47 |
+
if scenario_key == "before_commit" and attempt == 1:
|
| 48 |
+
events.append(Event(f"T+{attempt*20+5:03d}ms", "failure", "Worker stopped before the side effect was committed.", "bad").__dict__)
|
| 49 |
+
continue
|
| 50 |
+
if scenario_key == "transient" and attempt == 1:
|
| 51 |
+
events.append(Event(f"T+{attempt*20+5:03d}ms", "failure", "Downstream returned a retryable 503; no effect was committed.", "bad").__dict__)
|
| 52 |
+
continue
|
| 53 |
+
|
| 54 |
+
if mode_key == "naive":
|
| 55 |
+
side_effects += 1
|
| 56 |
+
events.append(Event(f"T+{attempt*20+8:03d}ms", "side effect", f"Charge applied. Total applied: {side_effects}.", "good").__dict__)
|
| 57 |
+
else:
|
| 58 |
+
prior = next((row for row in ledger if row["key"] == key), None)
|
| 59 |
+
if prior:
|
| 60 |
+
events.append(Event(f"T+{attempt*20+8:03d}ms", "deduplicated", f"Key already completed; replay cached result {prior['result']}.", "good").__dict__)
|
| 61 |
+
else:
|
| 62 |
+
side_effects += 1
|
| 63 |
+
receipt = "ch_042" if mode_key == "idempotent" else "evt_042"
|
| 64 |
+
ledger.append({"key": key, "result": receipt, "status": "completed"})
|
| 65 |
+
events.append(Event(f"T+{attempt*20+8:03d}ms", "side effect", f"Effect committed once; receipt {receipt}.", "good").__dict__)
|
| 66 |
+
|
| 67 |
+
if scenario_key == "commit_lost" and attempt == 1:
|
| 68 |
+
events.append(Event(f"T+{attempt*20+12:03d}ms", "failure", "Worker lost the response after commit; caller cannot tell whether it succeeded.", "bad").__dict__)
|
| 69 |
+
continue
|
| 70 |
+
events.append(Event(f"T+{attempt*20+15:03d}ms", "response", "200 OK · operation complete", "good").__dict__)
|
| 71 |
+
break
|
| 72 |
+
else:
|
| 73 |
+
events.append(Event("T+final", "queue", "Retry limit reached; work remains recoverable.", "bad").__dict__)
|
| 74 |
+
|
| 75 |
+
if scenario_key == "commit_lost":
|
| 76 |
+
explanation = ("A lost response creates an ambiguous outcome. The naive retry applies the charge twice; "
|
| 77 |
+
"an idempotency ledger returns the original result, while an outbox keeps durable work queued.")
|
| 78 |
+
elif scenario_key == "before_commit":
|
| 79 |
+
explanation = "No effect happened before the crash. Retrying is safe here; production systems still need bounded retries and durable state."
|
| 80 |
+
else:
|
| 81 |
+
explanation = "A transient 503 with no committed effect is retryable. Backoff and jitter would spread load in a real worker."
|
| 82 |
+
status = "Duplicate side effect" if side_effects > 1 else ("Completed once" if side_effects else "No side effect")
|
| 83 |
+
return Result(mode, scenario, attempts, side_effects, status, ledger, events, explanation)
|
index.html
CHANGED
|
@@ -1,19 +1,58 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
| 5 |
+
<meta name="description" content="Interactive simulator for retry safety, idempotency keys, and transactional outbox patterns.">
|
| 6 |
+
<title>Async Task Reliability Lab</title>
|
| 7 |
+
<style>
|
| 8 |
+
:root{color-scheme:dark;--bg:#0b1220;--panel:#121d2e;--panel2:#172438;--line:#293a50;--text:#e9f0f8;--muted:#9aabc0;--teal:#69ddc3;--blue:#80b9ff;--red:#ff7f86;--amber:#f3c77d;--green:#90e0ae}*{box-sizing:border-box}body{margin:0;background:radial-gradient(ellipse at 80% -10%,#213b53 0,transparent 38%),var(--bg);color:var(--text);font:15px/1.55 Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}.wrap{max-width:1100px;margin:auto;padding:38px 22px 70px}.hero{padding:28px 30px;border:1px solid #31445b;border-radius:22px;background:linear-gradient(135deg,#172b42,#172338 65%,#20374b);box-shadow:0 18px 55px #02081566}.eyebrow{text-transform:uppercase;letter-spacing:.16em;color:var(--teal);font-weight:700;font-size:11px}.hero h1{font-size:clamp(30px,5vw,48px);line-height:1.05;margin:10px 0 12px;letter-spacing:-.04em}.hero p{color:#bdcce0;max-width:760px;margin:0;font-size:16px}.hero strong{color:white}.grid{display:grid;grid-template-columns:340px 1fr;gap:18px;margin-top:18px}.panel{background:linear-gradient(160deg,#152236,#111b2b);border:1px solid var(--line);border-radius:18px;padding:20px;box-shadow:0 12px 32px #0002}.panel h2{font-size:16px;margin:0 0 16px}.field{margin:0 0 17px}.field label{display:block;font-size:12px;color:var(--muted);margin-bottom:7px}.field select{width:100%;background:#0d1726;color:var(--text);border:1px solid #344860;border-radius:10px;padding:11px 12px;font:inherit}.field select:focus,button:focus{outline:2px solid var(--teal);outline-offset:2px}.rangehead{display:flex;justify-content:space-between;align-items:center}.rangehead output{color:var(--teal);font-weight:700}.field input[type=range]{width:100%;accent-color:var(--teal);margin-top:11px}.hint{font-size:12px;color:var(--muted);margin:-6px 0 17px}.run{width:100%;border:0;border-radius:10px;background:var(--teal);color:#07151a;padding:12px 16px;font-weight:800;font-size:14px;cursor:pointer}.run:hover{filter:brightness(1.08)}.results{display:flex;flex-direction:column;gap:14px}.summary{display:flex;align-items:center;gap:14px;min-height:94px}.badge{width:48px;height:48px;border-radius:50%;display:grid;place-items:center;background:#203748;font-size:23px;flex:none}.summary h2{margin:0 0 2px;font-size:19px}.summary p{margin:0;color:var(--muted)}.metrics{display:flex;gap:9px;flex-wrap:wrap;margin-top:9px}.metric{border:1px solid var(--line);background:#0e1928;padding:5px 9px;border-radius:8px;color:#cbd8e8;font-size:12px}.metric b{color:white}.explanation{color:#bfccdc;margin:0;border-left:2px solid var(--blue);padding-left:12px}.tablewrap{overflow-x:auto}.tablewrap h2{margin-bottom:10px}.table{width:100%;border-collapse:collapse;font-size:13px}.table th{text-align:left;color:var(--muted);font-weight:600;font-size:11px;text-transform:uppercase;letter-spacing:.08em;padding:8px;border-bottom:1px solid var(--line)}.table td{padding:9px 8px;border-bottom:1px solid #26364a;vertical-align:top}.table tr:last-child td{border-bottom:0}.time{font-variant-numeric:tabular-nums;color:#b4c5d9;white-space:nowrap}.kind{font-weight:700;white-space:nowrap}.tone-good .kind{color:var(--green)}.tone-bad .kind{color:var(--red)}.tone-info .kind{color:var(--blue)}.ledger-empty{color:var(--muted);font-size:13px}.foot{margin-top:18px;color:#8fa1b7;font-size:12px}.foot strong{color:#c5d2e1}.method{margin-top:18px;display:grid;grid-template-columns:repeat(3,1fr);gap:12px}.method article{border:1px solid var(--line);border-radius:14px;padding:14px;background:#101a29}.method h3{font-size:13px;margin:0 0 5px;color:var(--teal)}.method p{font-size:12px;color:var(--muted);margin:0}.sr{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}@media(max-width:800px){.grid{grid-template-columns:1fr}.wrap{padding:20px 14px 45px}.hero{padding:23px 20px}.method{grid-template-columns:1fr}.panel{padding:17px}}
|
| 9 |
+
</style>
|
| 10 |
+
</head>
|
| 11 |
+
<body><main class="wrap">
|
| 12 |
+
<header class="hero"><div class="eyebrow">Backend reliability · interactive lab</div><h1>Async Task Reliability Lab</h1><p>See why retries can duplicate work—and how <strong>idempotency keys</strong> and a <strong>transactional outbox</strong> change the failure story.</p></header>
|
| 13 |
+
<div class="grid">
|
| 14 |
+
<section class="panel" aria-labelledby="controls-title"><h2 id="controls-title">Configure a failure</h2>
|
| 15 |
+
<div class="field"><label for="strategy">Reliability strategy</label><select id="strategy"><option>Naive retry</option><option>Idempotency key</option><option>Transactional outbox</option></select></div>
|
| 16 |
+
<div class="field"><label for="scenario">Failure scenario</label><select id="scenario"><option value="commit_lost">Crash after commit (response lost)</option><option value="before_commit">Worker crashes before commit</option><option value="transient">Transient delivery failure</option></select></div>
|
| 17 |
+
<div class="field"><div class="rangehead"><label for="retries">Retries after first attempt</label><output id="retry-value" for="retries">3</output></div><input id="retries" type="range" min="0" max="5" step="1" value="3"></div>
|
| 18 |
+
<p class="hint">One synthetic request · deterministic outcomes · no real services</p><button class="run" id="run">Run failure simulation</button></section>
|
| 19 |
+
<div class="results" aria-live="polite">
|
| 20 |
+
<section class="panel summary"><div id="badge" class="badge" aria-hidden="true">🟢</div><div><h2 id="verdict">Completed once</h2><p id="stats">Attempts: 2 · Effects committed: 1</p><div class="metrics"><span class="metric">Strategy: <b id="chosen-strategy">Naive retry</b></span><span class="metric">Scenario: <b id="chosen-scenario">Commit lost</b></span></div></div></section>
|
| 21 |
+
<section class="panel"><h2>What happened</h2><p id="explanation" class="explanation"></p></section>
|
| 22 |
+
<section class="panel tablewrap"><h2>Event timeline</h2><table class="table"><thead><tr><th>Time</th><th>Event</th><th>Detail</th></tr></thead><tbody id="events"></tbody></table></section>
|
| 23 |
+
<section class="panel tablewrap"><h2>Idempotency / outbox ledger</h2><div id="ledger" class="ledger-empty"></div></section>
|
| 24 |
+
</div></div>
|
| 25 |
+
<section class="method" aria-label="Reliability patterns"><article><h3>Naive retries</h3><p>Repeat a side effect after an ambiguous response, risking duplicate work.</p></article><article><h3>Idempotency key</h3><p>Store a result by request key so a retry can replay the first outcome.</p></article><article><h3>Transactional outbox</h3><p>Commit domain state and a durable event together; delivery and consumer deduplication still need real infrastructure.</p></article></section>
|
| 26 |
+
<p class="foot"><strong>Scope:</strong> teaching simulation only. It uses in-memory synthetic records, not a database or payment service; its deterministic counts are examples, not production benchmarks. A real outbox needs transactional storage, a poller, observability, and recovery procedures.</p>
|
| 27 |
+
</main>
|
| 28 |
+
<script>
|
| 29 |
+
const $=id=>document.getElementById(id), retries=$('retries');
|
| 30 |
+
retries.addEventListener('input',()=> $('retry-value').textContent=retries.value);
|
| 31 |
+
function simulate(){
|
| 32 |
+
const mode=$('strategy').value, scenario=$('scenario').value, max=Number(retries.value), key='order-7f2a';
|
| 33 |
+
let events=[], ledger=[], effects=0, attempts=0;
|
| 34 |
+
const ev=(t,k,d,tone='info')=>events.push({t,k,d,tone});
|
| 35 |
+
ev('T+00ms','request',`POST /charge · idempotency-key: ${key}`);
|
| 36 |
+
if(mode==='Transactional outbox')ev('T+02ms','database','Order + outbox event committed atomically.','good');
|
| 37 |
+
for(let attempt=1;attempt<=max+1;attempt++){
|
| 38 |
+
attempts=attempt; const t=attempt*20; ev(`T+${String(t).padStart(3,'0')}ms`,'worker',`Attempt ${attempt}: handling ${key}.`);
|
| 39 |
+
if(scenario==='before_commit'&&attempt===1){ev(`T+${t+5}ms`,'failure','Worker stopped before the side effect was committed.','bad');continue;}
|
| 40 |
+
if(scenario==='transient'&&attempt===1){ev(`T+${t+5}ms`,'failure','Downstream returned a retryable 503; no effect was committed.','bad');continue;}
|
| 41 |
+
if(mode==='Naive retry'){
|
| 42 |
+
effects++; ev(`T+${t+8}ms`,'side effect',`Charge applied. Total applied: ${effects}.`,'good');
|
| 43 |
+
}else{
|
| 44 |
+
const prior=ledger.find(x=>x.key===key);
|
| 45 |
+
if(prior)ev(`T+${t+8}ms`,'deduplicated',`Key already completed; replay cached result ${prior.result}.`,'good');
|
| 46 |
+
else{effects++;const receipt=mode==='Idempotency key'?'ch_042':'evt_042';ledger.push({key,result:receipt,status:'completed'});ev(`T+${t+8}ms`,'side effect',`Effect committed once; receipt ${receipt}.`,'good');}
|
| 47 |
+
}
|
| 48 |
+
if(scenario==='commit_lost'&&attempt===1){ev(`T+${t+12}ms`,'failure','Worker lost the response after commit; caller cannot tell whether it succeeded.','bad');continue;}
|
| 49 |
+
ev(`T+${t+15}ms`,'response','200 OK · operation complete','good');break;
|
| 50 |
+
}
|
| 51 |
+
let explanation=scenario==='commit_lost'?'A lost response creates an ambiguous outcome. The naive retry applies the charge twice; an idempotency ledger replays the original result, while an outbox makes work durable for delivery.':scenario==='before_commit'?'No effect happened before the crash. Retrying is safe in this example; production systems still need bounded retries and durable state.':'A transient 503 with no committed effect is retryable. Backoff and jitter would spread load in a real worker.';
|
| 52 |
+
const status=effects>1?'Duplicate side effect':effects===1?'Completed once':'No side effect';
|
| 53 |
+
$('verdict').textContent=status;$('stats').textContent=`Attempts: ${attempts} · Effects committed: ${effects}`;$('chosen-strategy').textContent=mode;$('chosen-scenario').textContent=$('scenario').selectedOptions[0].textContent;$('explanation').textContent=explanation;
|
| 54 |
+
$('badge').textContent=effects>1?'🔴':effects===1?'🟢':'🟡';$('events').innerHTML=events.map(e=>`<tr class="tone-${e.tone}"><td class="time">${e.t}</td><td class="kind">${e.k}</td><td>${e.d}</td></tr>`).join('');
|
| 55 |
+
$('ledger').innerHTML=ledger.length?`<table class="table"><thead><tr><th>Key</th><th>Result</th><th>Status</th></tr></thead><tbody>${ledger.map(r=>`<tr><td>${r.key}</td><td>${r.result}</td><td>${r.status}</td></tr>`).join('')}</tbody></table>`:'No deduplication record in this strategy.';
|
| 56 |
+
}
|
| 57 |
+
$('run').addEventListener('click',simulate);simulate();
|
| 58 |
+
</script></body></html>
|
test_engine.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import unittest
|
| 2 |
+
from engine import simulate
|
| 3 |
+
|
| 4 |
+
class ReliabilityTests(unittest.TestCase):
|
| 5 |
+
def test_naive_retry_duplicates_after_lost_response(self):
|
| 6 |
+
result = simulate("Naive retry", "Crash after commit (response lost)")
|
| 7 |
+
self.assertEqual(result.side_effects, 2)
|
| 8 |
+
self.assertEqual(result.status, "Duplicate side effect")
|
| 9 |
+
|
| 10 |
+
def test_idempotency_replays_original_result(self):
|
| 11 |
+
result = simulate("Idempotency key", "Crash after commit (response lost)")
|
| 12 |
+
self.assertEqual(result.side_effects, 1)
|
| 13 |
+
self.assertEqual(len(result.ledger), 1)
|
| 14 |
+
self.assertEqual(sum(e["kind"] == "deduplicated" for e in result.events), 1)
|
| 15 |
+
|
| 16 |
+
def test_outbox_keeps_single_durable_event(self):
|
| 17 |
+
result = simulate("Transactional outbox", "Crash after commit (response lost)")
|
| 18 |
+
self.assertEqual(result.side_effects, 1)
|
| 19 |
+
self.assertEqual(result.ledger[0]["status"], "completed")
|
| 20 |
+
self.assertTrue(any("outbox event committed" in e["detail"] for e in result.events))
|
| 21 |
+
|
| 22 |
+
def test_precommit_crash_is_retryable(self):
|
| 23 |
+
result = simulate("Naive retry", "Worker crashes before commit")
|
| 24 |
+
self.assertEqual(result.side_effects, 1)
|
| 25 |
+
|
| 26 |
+
def test_retry_exhaustion_is_bounded(self):
|
| 27 |
+
result = simulate("Naive retry", "Worker crashes before commit", max_retries=0)
|
| 28 |
+
self.assertEqual(result.attempts, 1)
|
| 29 |
+
self.assertEqual(result.side_effects, 0)
|
| 30 |
+
|
| 31 |
+
if __name__ == "__main__":
|
| 32 |
+
unittest.main()
|