Spaces:
Sleeping
fix(drugs): one ChEMBL window for the mechanism lookups, retried round-robin
Browse filesThe 2026-10-29 -> 11-05 smoke week failed at 91.8 % on one probe: right after 00, 06, 12
and 18 UTC ChEMBL did not answer drugs_kras's three mechanism lookups within 30 s (26 of 57
runs). Each lookup waited its own 30 s, so a failing run took ~90 s and reported three
identical errors; the Open Targets rows were already returned. The request is not heavy
(/mechanism.json, 1.3 KB, < 1 s by hand): ChEMBL is slow then, from HF's network.
- drugs.py: the mechanism lookups share one window (MECHANISM_WINDOW_SECONDS, capped by
what is left of the endpoint budget). A source_unavailable lookup is retried round-robin
with the other pending drugs; a quick failure pauses 5 s first; a drift is not retried.
Whatever is unanswered when the window closes gets ONE chembl source_unavailable entry
("... not looked up for N of M drugs (k attempts in a W s window)") and looked_up: false.
- _common.py: Collector.attempt returns the error instead of recording it; every call's
deadline is now capped by the remaining endpoint budget, so 240 s per endpoint call holds
exactly (a call started near the end could run 30 s past it).
- tu_client.py: call(deadline=) shortens timeout_seconds, never extends it; 30 s per call
stands, so the security addendum's figures are unchanged.
- tests/test_chembl_window.py (8): the KRAS cassette with ChEMBL silent for N calls on a
fake clock; hold for any window value (checked at 30, 60, 90, 150).
Suite 327 passed; ruff clean; security scan green. Window value is Annie's call (150 s
recommended); not merged until then.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
- src/endpoints/_common.py +26 -11
- src/endpoints/drugs.py +88 -15
- src/tu_client.py +14 -3
- tests/test_chembl_window.py +214 -0
|
@@ -47,6 +47,7 @@ class Budget:
|
|
| 47 |
|
| 48 |
def __init__(self, seconds: float = ENDPOINT_BUDGET_SECONDS, clock=time.monotonic):
|
| 49 |
self._clock = clock
|
|
|
|
| 50 |
self._deadline = clock() + seconds
|
| 51 |
|
| 52 |
@property
|
|
@@ -86,22 +87,36 @@ class Collector:
|
|
| 86 |
"""
|
| 87 |
if budget.spent:
|
| 88 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
self.calls += 1
|
| 90 |
try:
|
| 91 |
-
result = client.call(
|
|
|
|
|
|
|
| 92 |
except SourceError as err:
|
| 93 |
-
|
| 94 |
-
unknown is not None
|
| 95 |
-
and err.reason == "source_unavailable"
|
| 96 |
-
and unknown[0] in err.detail
|
| 97 |
-
):
|
| 98 |
-
self.unresolved_input(unknown[1], f"unknown to {err.source}: {err.detail}")
|
| 99 |
-
else:
|
| 100 |
-
self.source_errors.append(err.as_payload())
|
| 101 |
-
return None
|
| 102 |
self.sources_used.add(result.source)
|
| 103 |
self.retrieved.append(result.retrieved_at)
|
| 104 |
-
return result
|
| 105 |
|
| 106 |
def unresolved_input(self, value: str, reason: str) -> None:
|
| 107 |
self.unresolved.append({"input": value, "reason": reason})
|
|
|
|
| 47 |
|
| 48 |
def __init__(self, seconds: float = ENDPOINT_BUDGET_SECONDS, clock=time.monotonic):
|
| 49 |
self._clock = clock
|
| 50 |
+
self.seconds = seconds
|
| 51 |
self._deadline = clock() + seconds
|
| 52 |
|
| 53 |
@property
|
|
|
|
| 87 |
"""
|
| 88 |
if budget.spent:
|
| 89 |
return None
|
| 90 |
+
result, err = self.attempt(client, budget, tool, args)
|
| 91 |
+
if err is None:
|
| 92 |
+
return result
|
| 93 |
+
if unknown is not None and err.reason == "source_unavailable" and unknown[0] in err.detail:
|
| 94 |
+
self.unresolved_input(unknown[1], f"unknown to {err.source}: {err.detail}")
|
| 95 |
+
else:
|
| 96 |
+
self.source_errors.append(err.as_payload())
|
| 97 |
+
return None
|
| 98 |
+
|
| 99 |
+
def attempt(
|
| 100 |
+
self,
|
| 101 |
+
client: ToolClient,
|
| 102 |
+
budget: Budget,
|
| 103 |
+
tool: str,
|
| 104 |
+
args: dict[str, Any],
|
| 105 |
+
) -> tuple[SourceResult | None, SourceError | None]:
|
| 106 |
+
"""One source call whose failure is handed back, not recorded: the caller may retry it.
|
| 107 |
+
|
| 108 |
+
The call's deadline never runs past ``budget``: a call started with 12 s left gets 12 s.
|
| 109 |
+
"""
|
| 110 |
self.calls += 1
|
| 111 |
try:
|
| 112 |
+
result = client.call(
|
| 113 |
+
tool, args, max_wait=min(10.0, budget.remaining), deadline=budget.remaining
|
| 114 |
+
)
|
| 115 |
except SourceError as err:
|
| 116 |
+
return None, err
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
self.sources_used.add(result.source)
|
| 118 |
self.retrieved.append(result.retrieved_at)
|
| 119 |
+
return result, None
|
| 120 |
|
| 121 |
def unresolved_input(self, value: str, reason: str) -> None:
|
| 122 |
self.unresolved.append({"input": value, "reason": reason})
|
|
@@ -6,13 +6,24 @@ mechanism lookup is one call per drug, so it is capped at the ``MAX_MECHANISM_LO
|
|
| 6 |
advanced drugs in the call (stage first, then name); the payload says how many were looked up.
|
| 7 |
Open Targets' coverage note travels with every payload: an investigational compound with no
|
| 8 |
trial record can be absent (Phase 0: MRTX1133 against KRAS G12D).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
from __future__ import annotations
|
| 12 |
|
|
|
|
| 13 |
from typing import Any
|
| 14 |
|
| 15 |
-
from src import payloads
|
| 16 |
from src.endpoints._common import (
|
| 17 |
Budget,
|
| 18 |
Collector,
|
|
@@ -26,12 +37,25 @@ from src.endpoints._common import (
|
|
| 26 |
stamped,
|
| 27 |
)
|
| 28 |
from src.identifiers import parse_gene_list
|
|
|
|
| 29 |
|
| 30 |
ENDPOINT = "drugs_for_targets"
|
| 31 |
OT_DRUGS = "OpenTargets_get_associated_drugs_by_target_ensemblID"
|
| 32 |
CHEMBL_MECHANISMS = "ChEMBL_get_drug_mechanisms"
|
| 33 |
MAX_MECHANISM_LOOKUPS = 30
|
| 34 |
MAX_INDICATIONS = 10
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
def drugs_for_targets(genes: str = "") -> dict[str, Any]:
|
|
@@ -126,27 +150,76 @@ def _attach_mechanisms(out: Collector, client, budget: Budget) -> int:
|
|
| 126 |
for row in out.rows:
|
| 127 |
key = (stage_rank(row["max_clinical_stage"]), row["drug"])
|
| 128 |
best[row["drug_chembl_id"]] = min(best.get(row["drug_chembl_id"], key), key)
|
| 129 |
-
order = sorted(best, key=lambda chembl_id: (best[chembl_id], chembl_id))
|
|
|
|
| 130 |
mechanisms: dict[str, dict[str, Any]] = {}
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
for row in out.rows:
|
| 145 |
found = mechanisms.get(row["drug_chembl_id"])
|
| 146 |
row["mechanisms"] = found if found is not None else {"looked_up": False}
|
| 147 |
return len(mechanisms)
|
| 148 |
|
| 149 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
def _narrative(out: Collector, per_target: list[tuple[str, int]], looked_up: int) -> str:
|
| 151 |
sentences = []
|
| 152 |
clauses = []
|
|
|
|
| 6 |
advanced drugs in the call (stage first, then name); the payload says how many were looked up.
|
| 7 |
Open Targets' coverage note travels with every payload: an investigational compound with no
|
| 8 |
trial record can be absent (Phase 0: MRTX1133 against KRAS G12D).
|
| 9 |
+
|
| 10 |
+
The mechanism lookups share one ChEMBL window per call (``MECHANISM_WINDOW_SECONDS``) instead of
|
| 11 |
+
30 s each, and a lookup ChEMBL does not answer is tried again inside it, round-robin with the
|
| 12 |
+
others still waiting. Why: over the 2026-10-29 → 11-05 smoke week, ChEMBL stopped answering
|
| 13 |
+
within 30 s right after 00, 06, 12 and 18 UTC (26 of 57 KRAS probes; the other sources answered
|
| 14 |
+
98-100 % of the week), while the request itself is a 1.3 KB answer in under a second at other
|
| 15 |
+
times (timed by hand 2026-11-05). A slow
|
| 16 |
+
minute is the source's state, not one drug's, so waiting it out is the only thing that recovers
|
| 17 |
+
the answer. Whatever is still unanswered when the window closes is named once in
|
| 18 |
+
``source_errors`` and its rows say ``looked_up: false``; the Open Targets rows always stand.
|
| 19 |
"""
|
| 20 |
|
| 21 |
from __future__ import annotations
|
| 22 |
|
| 23 |
+
import time
|
| 24 |
from typing import Any
|
| 25 |
|
| 26 |
+
from src import config, payloads
|
| 27 |
from src.endpoints._common import (
|
| 28 |
Budget,
|
| 29 |
Collector,
|
|
|
|
| 37 |
stamped,
|
| 38 |
)
|
| 39 |
from src.identifiers import parse_gene_list
|
| 40 |
+
from src.tu_client import SourceError
|
| 41 |
|
| 42 |
ENDPOINT = "drugs_for_targets"
|
| 43 |
OT_DRUGS = "OpenTargets_get_associated_drugs_by_target_ensemblID"
|
| 44 |
CHEMBL_MECHANISMS = "ChEMBL_get_drug_mechanisms"
|
| 45 |
MAX_MECHANISM_LOOKUPS = 30
|
| 46 |
MAX_INDICATIONS = 10
|
| 47 |
+
# All of one call's ChEMBL mechanism lookups, retries included, finish inside this window (and
|
| 48 |
+
# inside what is left of the endpoint's own budget). Each single call still stops at 30 s.
|
| 49 |
+
MECHANISM_WINDOW_SECONDS = 150
|
| 50 |
+
# After a quick failure (an error answer, not silence) the next attempt waits this long, so a
|
| 51 |
+
# struggling source is not hammered. A timeout has already waited its 30 s and does not pause.
|
| 52 |
+
RETRY_PAUSE_SECONDS = 5
|
| 53 |
+
# A backstop for a lookup that keeps failing quickly; the window is the real bound.
|
| 54 |
+
MAX_ATTEMPTS_PER_LOOKUP = 5
|
| 55 |
+
|
| 56 |
+
# The mechanism window's clock and sleep, swapped for a fake clock in tests.
|
| 57 |
+
_clock = time.monotonic
|
| 58 |
+
_sleep = time.sleep
|
| 59 |
|
| 60 |
|
| 61 |
def drugs_for_targets(genes: str = "") -> dict[str, Any]:
|
|
|
|
| 150 |
for row in out.rows:
|
| 151 |
key = (stage_rank(row["max_clinical_stage"]), row["drug"])
|
| 152 |
best[row["drug_chembl_id"]] = min(best.get(row["drug_chembl_id"], key), key)
|
| 153 |
+
order = sorted(best, key=lambda chembl_id: (best[chembl_id], chembl_id))[:MAX_MECHANISM_LOOKUPS]
|
| 154 |
+
window = Budget(min(MECHANISM_WINDOW_SECONDS, budget.remaining), clock=_clock)
|
| 155 |
mechanisms: dict[str, dict[str, Any]] = {}
|
| 156 |
+
tries = dict.fromkeys(order, 0)
|
| 157 |
+
last_failure: SourceError | None = None
|
| 158 |
+
pending = list(order)
|
| 159 |
+
while pending and not window.spent:
|
| 160 |
+
waiting = []
|
| 161 |
+
for chembl_id in pending:
|
| 162 |
+
if window.spent or tries[chembl_id] >= MAX_ATTEMPTS_PER_LOOKUP:
|
| 163 |
+
waiting.append(chembl_id)
|
| 164 |
+
continue
|
| 165 |
+
tries[chembl_id] += 1
|
| 166 |
+
started = _clock()
|
| 167 |
+
result, err = out.attempt(
|
| 168 |
+
client, window, CHEMBL_MECHANISMS, {"drug_chembl_id": chembl_id}
|
| 169 |
+
)
|
| 170 |
+
if err is not None:
|
| 171 |
+
if err.reason != "source_unavailable":
|
| 172 |
+
out.source_errors.append(err.as_payload()) # a drift is not retried
|
| 173 |
+
continue
|
| 174 |
+
last_failure = err
|
| 175 |
+
waiting.append(chembl_id)
|
| 176 |
+
if _clock() - started < RETRY_PAUSE_SECONDS and not window.spent:
|
| 177 |
+
_sleep(min(RETRY_PAUSE_SECONDS, window.remaining))
|
| 178 |
+
continue
|
| 179 |
+
records = result.data.get("mechanisms") or []
|
| 180 |
+
moa = sorted(
|
| 181 |
+
{m["mechanism_of_action"] for m in records if m.get("mechanism_of_action")}
|
| 182 |
+
)
|
| 183 |
+
actions = sorted({m["action_type"] for m in records if m.get("action_type")})
|
| 184 |
+
mechanisms[chembl_id] = stamped(
|
| 185 |
+
{"mechanism_of_action": moa, "action_types": actions},
|
| 186 |
+
result,
|
| 187 |
+
ref=chembl_id,
|
| 188 |
+
url=f"https://www.ebi.ac.uk/chembl/compound_report_card/{chembl_id}/",
|
| 189 |
+
)
|
| 190 |
+
pending = waiting
|
| 191 |
+
if all(tries[c] >= MAX_ATTEMPTS_PER_LOOKUP for c in pending):
|
| 192 |
+
break
|
| 193 |
+
if pending:
|
| 194 |
+
out.source_errors.append(_unanswered(pending, order, tries, last_failure, window))
|
| 195 |
for row in out.rows:
|
| 196 |
found = mechanisms.get(row["drug_chembl_id"])
|
| 197 |
row["mechanisms"] = found if found is not None else {"looked_up": False}
|
| 198 |
return len(mechanisms)
|
| 199 |
|
| 200 |
|
| 201 |
+
def _unanswered(
|
| 202 |
+
pending: list[str],
|
| 203 |
+
order: list[str],
|
| 204 |
+
tries: dict[str, int],
|
| 205 |
+
last_failure: SourceError | None,
|
| 206 |
+
window: Budget,
|
| 207 |
+
) -> dict[str, str]:
|
| 208 |
+
"""One ``source_errors`` entry for every mechanism lookup ChEMBL left unanswered."""
|
| 209 |
+
attempts = sum(tries.values())
|
| 210 |
+
how = (
|
| 211 |
+
last_failure.detail
|
| 212 |
+
if last_failure is not None
|
| 213 |
+
else f"not looked up: the {ENDPOINT} call budget ran out"
|
| 214 |
+
)
|
| 215 |
+
return SourceError(
|
| 216 |
+
config.source_of(CHEMBL_MECHANISMS),
|
| 217 |
+
"source_unavailable",
|
| 218 |
+
f"{how}; mechanisms not looked up for {len(pending)} of {len(order)} drugs "
|
| 219 |
+
f"({attempts} attempt{'s' if attempts != 1 else ''} in a {round(window.seconds):g} s window)",
|
| 220 |
+
).as_payload()
|
| 221 |
+
|
| 222 |
+
|
| 223 |
def _narrative(out: Collector, per_target: list[tuple[str, int]], looked_up: int) -> str:
|
| 224 |
sentences = []
|
| 225 |
clauses = []
|
|
@@ -290,7 +290,15 @@ class ToolClient:
|
|
| 290 |
self._tu = build_tooluniverse()
|
| 291 |
return self._tu.run_one_function({"name": tool, "arguments": args}, use_cache=False)
|
| 292 |
|
| 293 |
-
def call(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
if tool not in config.endpoint_tool_names():
|
| 295 |
# A programming error, not a user input: endpoints only ever name loaded tools.
|
| 296 |
raise ValueError(f"{tool!r} is not an endpoint tool")
|
|
@@ -309,18 +317,21 @@ class ToolClient:
|
|
| 309 |
source, "rate_limited", f"{cfg['name']} limit of {cfg['rate_per_second']}/s reached"
|
| 310 |
)
|
| 311 |
|
|
|
|
|
|
|
|
|
|
| 312 |
runner = self._runner or self._default_runner
|
| 313 |
future = self._pool.submit(runner, tool, dict(args))
|
| 314 |
# `wait`, not `result(timeout=…)`: since Python 3.11 concurrent.futures.TimeoutError IS the
|
| 315 |
# builtin TimeoutError, so a source's own read timeout raised inside the call would be
|
| 316 |
# indistinguishable from our deadline passing (caught by a test on 2026-10-20).
|
| 317 |
-
done, _ = concurrent.futures.wait([future], timeout=
|
| 318 |
if not done:
|
| 319 |
future.cancel()
|
| 320 |
raise SourceError(
|
| 321 |
source,
|
| 322 |
"source_unavailable",
|
| 323 |
-
f"{cfg['name']} did not answer within {
|
| 324 |
)
|
| 325 |
try:
|
| 326 |
raw = future.result()
|
|
|
|
| 290 |
self._tu = build_tooluniverse()
|
| 291 |
return self._tu.run_one_function({"name": tool, "arguments": args}, use_cache=False)
|
| 292 |
|
| 293 |
+
def call(
|
| 294 |
+
self,
|
| 295 |
+
tool: str,
|
| 296 |
+
args: dict[str, Any],
|
| 297 |
+
*,
|
| 298 |
+
max_wait: float = 10.0,
|
| 299 |
+
deadline: float | None = None,
|
| 300 |
+
) -> SourceResult:
|
| 301 |
+
"""``deadline`` can shorten the source's ``timeout_seconds`` for this call, never extend it."""
|
| 302 |
if tool not in config.endpoint_tool_names():
|
| 303 |
# A programming error, not a user input: endpoints only ever name loaded tools.
|
| 304 |
raise ValueError(f"{tool!r} is not an endpoint tool")
|
|
|
|
| 317 |
source, "rate_limited", f"{cfg['name']} limit of {cfg['rate_per_second']}/s reached"
|
| 318 |
)
|
| 319 |
|
| 320 |
+
timeout = cfg["timeout_seconds"]
|
| 321 |
+
if deadline is not None:
|
| 322 |
+
timeout = max(0.0, min(timeout, deadline))
|
| 323 |
runner = self._runner or self._default_runner
|
| 324 |
future = self._pool.submit(runner, tool, dict(args))
|
| 325 |
# `wait`, not `result(timeout=…)`: since Python 3.11 concurrent.futures.TimeoutError IS the
|
| 326 |
# builtin TimeoutError, so a source's own read timeout raised inside the call would be
|
| 327 |
# indistinguishable from our deadline passing (caught by a test on 2026-10-20).
|
| 328 |
+
done, _ = concurrent.futures.wait([future], timeout=timeout)
|
| 329 |
if not done:
|
| 330 |
future.cancel()
|
| 331 |
raise SourceError(
|
| 332 |
source,
|
| 333 |
"source_unavailable",
|
| 334 |
+
f"{cfg['name']} did not answer within {round(timeout, 1):g} s",
|
| 335 |
)
|
| 336 |
try:
|
| 337 |
raw = future.result()
|
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ChEMBL's slow window, offline: the KRAS probe's recording with ChEMBL made silent for a while.
|
| 2 |
+
|
| 3 |
+
The 2026-10-29 → 11-05 smoke week failed on one thing: right after 00, 06, 12 and 18 UTC ChEMBL did
|
| 4 |
+
not answer the KRAS probe's three mechanism lookups within 30 s (26 of 57 runs), and each lookup
|
| 5 |
+
waited its own 30 s, so a failing run took ~90 s and reported three identical errors. These tests
|
| 6 |
+
replay the recorded KRAS scenario (``q1_q2_drugs_kras``) on a fake clock, with ChEMBL silent for its
|
| 7 |
+
first N calls, and pin what the endpoint does now: it waits the silence out inside one shared
|
| 8 |
+
window, and when ChEMBL stays silent the Open Targets rows still stand, every row says its
|
| 9 |
+
mechanisms were not looked up, and ONE structured error names ChEMBL.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import time
|
| 15 |
+
|
| 16 |
+
import pytest
|
| 17 |
+
|
| 18 |
+
from scripts import smoke_live as sl
|
| 19 |
+
from src import config
|
| 20 |
+
from src.cache import ResponseCache
|
| 21 |
+
from src.endpoints import _common, drugs
|
| 22 |
+
from src.ratelimit import RateLimiters
|
| 23 |
+
from src.tu_client import SourceError, ToolClient
|
| 24 |
+
from tests.conftest import CassetteRunner, FakeClock, FakeRunner
|
| 25 |
+
|
| 26 |
+
CASSETTE = "q1_q2_drugs_kras"
|
| 27 |
+
DEADLINE = 30 # ChEMBL's timeout_seconds: what one silent call costs
|
| 28 |
+
KRAS_PROBE = next(p for p in sl.PROBES if p.id == "drugs_kras")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class SilentChembl:
|
| 32 |
+
"""The recording, except ChEMBL's first ``silent`` calls fail the way production saw them."""
|
| 33 |
+
|
| 34 |
+
def __init__(self, clock: FakeClock, silent: int, *, quick: bool = False, drift: str = ""):
|
| 35 |
+
self.inner = CassetteRunner(CASSETTE)
|
| 36 |
+
self.clock = clock
|
| 37 |
+
self.silent = silent
|
| 38 |
+
self.quick = quick
|
| 39 |
+
self.drift = drift
|
| 40 |
+
self.chembl_calls: list[str] = []
|
| 41 |
+
|
| 42 |
+
def __call__(self, tool: str, args: dict):
|
| 43 |
+
if tool != drugs.CHEMBL_MECHANISMS:
|
| 44 |
+
return self.inner(tool, args)
|
| 45 |
+
self.chembl_calls.append(args["drug_chembl_id"])
|
| 46 |
+
if args["drug_chembl_id"] == self.drift:
|
| 47 |
+
return {"status": "success", "data": {"mechanism": []}} # no `mechanisms` key
|
| 48 |
+
if self.silent > 0:
|
| 49 |
+
self.silent -= 1
|
| 50 |
+
if self.quick:
|
| 51 |
+
raise ConnectionError("connection reset by peer")
|
| 52 |
+
self.clock.advance(DEADLINE)
|
| 53 |
+
# What the client raises when its deadline passes (src/tu_client.py).
|
| 54 |
+
raise SourceError("chembl", "source_unavailable", "ChEMBL did not answer within 30 s")
|
| 55 |
+
return self.inner(tool, args)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@pytest.fixture()
|
| 59 |
+
def kras(monkeypatch):
|
| 60 |
+
"""``kras(silent, ...)`` → (payload, runner, seconds of fake time the call took)."""
|
| 61 |
+
|
| 62 |
+
def run(silent: int, **kwargs):
|
| 63 |
+
meta = CassetteRunner(CASSETTE).cassette["_recorded"]
|
| 64 |
+
clock = FakeClock(start=meta["recorded_at_epoch"])
|
| 65 |
+
runner = SilentChembl(clock, silent, **kwargs)
|
| 66 |
+
monkeypatch.setattr(drugs, "_clock", clock)
|
| 67 |
+
monkeypatch.setattr(drugs, "_sleep", clock.sleep)
|
| 68 |
+
_common.set_client(
|
| 69 |
+
ToolClient(
|
| 70 |
+
runner=runner,
|
| 71 |
+
cache=ResponseCache(":memory:", clock=clock),
|
| 72 |
+
limiters=RateLimiters(clock=clock, sleep=clock.sleep),
|
| 73 |
+
clock=clock,
|
| 74 |
+
tooluniverse_version=meta["tooluniverse_version"],
|
| 75 |
+
)
|
| 76 |
+
)
|
| 77 |
+
started = clock.now
|
| 78 |
+
payload = drugs.drugs_for_targets(*meta["arguments"])
|
| 79 |
+
return payload, runner, clock.now - started, clock
|
| 80 |
+
|
| 81 |
+
yield run
|
| 82 |
+
_common.set_client(None)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _looked_up(payload) -> list[bool]:
|
| 86 |
+
return [r["mechanisms"].get("looked_up", True) for r in payload["rows"]]
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def test_the_recording_takes_one_call_per_drug(kras):
|
| 90 |
+
payload, runner, seconds, _ = kras(silent=0)
|
| 91 |
+
assert payload["source_errors"] == []
|
| 92 |
+
assert len(runner.chembl_calls) == 3 and seconds == 0
|
| 93 |
+
assert sl.evaluate(KRAS_PROBE, payload)[1] is None
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def test_a_slow_chembl_window_is_waited_out(kras):
|
| 97 |
+
"""Silent for all but the window's last call: before this fix, three errors and no mechanisms."""
|
| 98 |
+
silent = drugs.MECHANISM_WINDOW_SECONDS // DEADLINE - 1
|
| 99 |
+
payload, runner, seconds, _ = kras(silent=silent)
|
| 100 |
+
assert payload["status"] == "ok"
|
| 101 |
+
assert payload["source_errors"] == []
|
| 102 |
+
assert _looked_up(payload) == [True, True, True]
|
| 103 |
+
assert payload["mechanism_lookups"]["looked_up"] == 3
|
| 104 |
+
assert len(runner.chembl_calls) == silent + 3
|
| 105 |
+
assert seconds <= drugs.MECHANISM_WINDOW_SECONDS
|
| 106 |
+
assert sl.evaluate(KRAS_PROBE, payload)[1] is None # the smoke probe passes
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def test_silent_lookups_are_retried_round_robin(kras, monkeypatch):
|
| 110 |
+
"""A slow minute is ChEMBL's state, not one drug's: each pending drug is tried in turn."""
|
| 111 |
+
monkeypatch.setattr(drugs, "MECHANISM_WINDOW_SECONDS", 150) # room for five silent calls
|
| 112 |
+
_, runner, _, _ = kras(silent=4)
|
| 113 |
+
first_pass = runner.chembl_calls[:3]
|
| 114 |
+
assert sorted(first_pass) == sorted(set(first_pass)) # three different drugs first
|
| 115 |
+
assert runner.chembl_calls[3:5] == first_pass[:2] # then the first two again, in order
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def test_when_chembl_stays_silent_the_rows_stand_and_one_error_names_it(kras):
|
| 119 |
+
payload, runner, seconds, _ = kras(silent=10_000)
|
| 120 |
+
assert payload["status"] == "ok"
|
| 121 |
+
assert [r["drug"] for r in payload["rows"]] == ["ADAGRASIB", "SOTORASIB", "SALIRASIB"]
|
| 122 |
+
assert _looked_up(payload) == [False, False, False]
|
| 123 |
+
attempts = drugs.MECHANISM_WINDOW_SECONDS // DEADLINE
|
| 124 |
+
assert payload["source_errors"] == [
|
| 125 |
+
{
|
| 126 |
+
"source": "chembl",
|
| 127 |
+
"reason": "source_unavailable",
|
| 128 |
+
"detail": (
|
| 129 |
+
"ChEMBL did not answer within 30 s; mechanisms not looked up for 3 of 3 drugs "
|
| 130 |
+
f"({attempts} attempt{'s' if attempts != 1 else ''} in a "
|
| 131 |
+
f"{drugs.MECHANISM_WINDOW_SECONDS} s window)"
|
| 132 |
+
),
|
| 133 |
+
}
|
| 134 |
+
]
|
| 135 |
+
assert len(runner.chembl_calls) == attempts
|
| 136 |
+
assert seconds == drugs.MECHANISM_WINDOW_SECONDS
|
| 137 |
+
assert payload["narrative"].count("chembl source_unavailable") == 1
|
| 138 |
+
facts, failure = sl.evaluate(KRAS_PROBE, payload)
|
| 139 |
+
# Still a failed probe, but a structured one: the smoke measure's C4 holds.
|
| 140 |
+
assert failure["class"] == "source_unavailable" and failure["source"] == "chembl"
|
| 141 |
+
assert facts["n_rows"] == 3
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def test_a_quick_failure_pauses_before_the_next_attempt(kras):
|
| 145 |
+
payload, runner, _, clock = kras(silent=2, quick=True)
|
| 146 |
+
assert payload["source_errors"] == []
|
| 147 |
+
assert _looked_up(payload) == [True, True, True]
|
| 148 |
+
assert clock.sleeps == [drugs.RETRY_PAUSE_SECONDS] * 2
|
| 149 |
+
assert len(runner.chembl_calls) == 5
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def test_a_drifted_mechanism_answer_is_named_and_not_retried(kras):
|
| 153 |
+
payload, runner, _, _ = kras(silent=0, drift="CHEMBL23293")
|
| 154 |
+
assert runner.chembl_calls.count("CHEMBL23293") == 1
|
| 155 |
+
assert [e["reason"] for e in payload["source_errors"]] == ["upstream_schema_drift"]
|
| 156 |
+
assert sorted(_looked_up(payload)) == [False, True, True]
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def test_the_window_never_outlasts_what_is_left_of_the_endpoint_budget(monkeypatch):
|
| 160 |
+
monkeypatch.setattr(drugs, "MECHANISM_WINDOW_SECONDS", 150) # longer than the 40 s left
|
| 161 |
+
clock = FakeClock()
|
| 162 |
+
monkeypatch.setattr(drugs, "_clock", clock)
|
| 163 |
+
monkeypatch.setattr(drugs, "_sleep", clock.sleep)
|
| 164 |
+
deadlines: list[float] = []
|
| 165 |
+
|
| 166 |
+
class Client:
|
| 167 |
+
def call(self, tool, args, *, max_wait, deadline):
|
| 168 |
+
deadlines.append(deadline)
|
| 169 |
+
waited = min(DEADLINE, deadline) # the real client caps it at timeout_seconds
|
| 170 |
+
clock.advance(waited)
|
| 171 |
+
raise SourceError(
|
| 172 |
+
"chembl", "source_unavailable", f"ChEMBL did not answer within {waited:g} s"
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
out = _common.Collector()
|
| 176 |
+
out.rows = [
|
| 177 |
+
{"drug": name, "drug_chembl_id": cid, "max_clinical_stage": "PHASE_2"}
|
| 178 |
+
for name, cid in (("A", "CHEMBL1"), ("B", "CHEMBL2"), ("C", "CHEMBL3"))
|
| 179 |
+
]
|
| 180 |
+
assert drugs._attach_mechanisms(out, Client(), _common.Budget(40, clock=clock)) == 0
|
| 181 |
+
assert deadlines == [40, 10] # the second call gets only what is left
|
| 182 |
+
assert clock.now - FakeClock().now == 40
|
| 183 |
+
[error] = out.source_errors
|
| 184 |
+
assert error["detail"] == (
|
| 185 |
+
"ChEMBL did not answer within 10 s; mechanisms not looked up for 3 of 3 drugs "
|
| 186 |
+
"(2 attempts in a 40 s window)"
|
| 187 |
+
)
|
| 188 |
+
assert [r["mechanisms"] for r in out.rows] == [{"looked_up": False}] * 3
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def test_a_caller_deadline_shortens_but_never_extends_the_source_timeout(
|
| 192 |
+
monkeypatch, memory_cache, clock
|
| 193 |
+
):
|
| 194 |
+
def hangs(args):
|
| 195 |
+
time.sleep(1)
|
| 196 |
+
|
| 197 |
+
client = ToolClient(
|
| 198 |
+
runner=FakeRunner({drugs.CHEMBL_MECHANISMS: hangs}),
|
| 199 |
+
cache=memory_cache,
|
| 200 |
+
clock=clock,
|
| 201 |
+
tooluniverse_version="1.5.6",
|
| 202 |
+
)
|
| 203 |
+
started = time.monotonic()
|
| 204 |
+
with pytest.raises(SourceError) as shortened:
|
| 205 |
+
client.call(drugs.CHEMBL_MECHANISMS, {"drug_chembl_id": "CHEMBL1"}, deadline=0.2)
|
| 206 |
+
assert time.monotonic() - started < 0.9
|
| 207 |
+
assert shortened.value.detail == "ChEMBL did not answer within 0.2 s"
|
| 208 |
+
|
| 209 |
+
monkeypatch.setitem(config.source("chembl"), "timeout_seconds", 0.2)
|
| 210 |
+
started = time.monotonic()
|
| 211 |
+
with pytest.raises(SourceError) as capped:
|
| 212 |
+
client.call(drugs.CHEMBL_MECHANISMS, {"drug_chembl_id": "CHEMBL2"}, deadline=60)
|
| 213 |
+
assert time.monotonic() - started < 0.9
|
| 214 |
+
assert capped.value.detail == "ChEMBL did not answer within 0.2 s"
|