avoigt1121 Claude Opus 5.5 commited on
Commit
01b3ee5
·
1 Parent(s): d8d8c57

fix(drugs): one ChEMBL window for the mechanism lookups, retried round-robin

Browse files

The 2026-10-29 -> 11-05 smoke week failed at 91.8 % on one probe: right after 00, 06, 12
and 18 UTC ChEMBL did not answer drugs_kras's three mechanism lookups within 30 s (26 of 57
runs). Each lookup waited its own 30 s, so a failing run took ~90 s and reported three
identical errors; the Open Targets rows were already returned. The request is not heavy
(/mechanism.json, 1.3 KB, < 1 s by hand): ChEMBL is slow then, from HF's network.

- drugs.py: the mechanism lookups share one window (MECHANISM_WINDOW_SECONDS, capped by
what is left of the endpoint budget). A source_unavailable lookup is retried round-robin
with the other pending drugs; a quick failure pauses 5 s first; a drift is not retried.
Whatever is unanswered when the window closes gets ONE chembl source_unavailable entry
("... not looked up for N of M drugs (k attempts in a W s window)") and looked_up: false.
- _common.py: Collector.attempt returns the error instead of recording it; every call's
deadline is now capped by the remaining endpoint budget, so 240 s per endpoint call holds
exactly (a call started near the end could run 30 s past it).
- tu_client.py: call(deadline=) shortens timeout_seconds, never extends it; 30 s per call
stands, so the security addendum's figures are unchanged.
- tests/test_chembl_window.py (8): the KRAS cassette with ChEMBL silent for N calls on a
fake clock; hold for any window value (checked at 30, 60, 90, 150).

Suite 327 passed; ruff clean; security scan green. Window value is Annie's call (150 s
recommended); not merged until then.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/endpoints/_common.py CHANGED
@@ -47,6 +47,7 @@ class Budget:
47
 
48
  def __init__(self, seconds: float = ENDPOINT_BUDGET_SECONDS, clock=time.monotonic):
49
  self._clock = clock
 
50
  self._deadline = clock() + seconds
51
 
52
  @property
@@ -86,22 +87,36 @@ class Collector:
86
  """
87
  if budget.spent:
88
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  self.calls += 1
90
  try:
91
- result = client.call(tool, args, max_wait=min(10.0, budget.remaining))
 
 
92
  except SourceError as err:
93
- if (
94
- unknown is not None
95
- and err.reason == "source_unavailable"
96
- and unknown[0] in err.detail
97
- ):
98
- self.unresolved_input(unknown[1], f"unknown to {err.source}: {err.detail}")
99
- else:
100
- self.source_errors.append(err.as_payload())
101
- return None
102
  self.sources_used.add(result.source)
103
  self.retrieved.append(result.retrieved_at)
104
- return result
105
 
106
  def unresolved_input(self, value: str, reason: str) -> None:
107
  self.unresolved.append({"input": value, "reason": reason})
 
47
 
48
  def __init__(self, seconds: float = ENDPOINT_BUDGET_SECONDS, clock=time.monotonic):
49
  self._clock = clock
50
+ self.seconds = seconds
51
  self._deadline = clock() + seconds
52
 
53
  @property
 
87
  """
88
  if budget.spent:
89
  return None
90
+ result, err = self.attempt(client, budget, tool, args)
91
+ if err is None:
92
+ return result
93
+ if unknown is not None and err.reason == "source_unavailable" and unknown[0] in err.detail:
94
+ self.unresolved_input(unknown[1], f"unknown to {err.source}: {err.detail}")
95
+ else:
96
+ self.source_errors.append(err.as_payload())
97
+ return None
98
+
99
+ def attempt(
100
+ self,
101
+ client: ToolClient,
102
+ budget: Budget,
103
+ tool: str,
104
+ args: dict[str, Any],
105
+ ) -> tuple[SourceResult | None, SourceError | None]:
106
+ """One source call whose failure is handed back, not recorded: the caller may retry it.
107
+
108
+ The call's deadline never runs past ``budget``: a call started with 12 s left gets 12 s.
109
+ """
110
  self.calls += 1
111
  try:
112
+ result = client.call(
113
+ tool, args, max_wait=min(10.0, budget.remaining), deadline=budget.remaining
114
+ )
115
  except SourceError as err:
116
+ return None, err
 
 
 
 
 
 
 
 
117
  self.sources_used.add(result.source)
118
  self.retrieved.append(result.retrieved_at)
119
+ return result, None
120
 
121
  def unresolved_input(self, value: str, reason: str) -> None:
122
  self.unresolved.append({"input": value, "reason": reason})
src/endpoints/drugs.py CHANGED
@@ -6,13 +6,24 @@ mechanism lookup is one call per drug, so it is capped at the ``MAX_MECHANISM_LO
6
  advanced drugs in the call (stage first, then name); the payload says how many were looked up.
7
  Open Targets' coverage note travels with every payload: an investigational compound with no
8
  trial record can be absent (Phase 0: MRTX1133 against KRAS G12D).
 
 
 
 
 
 
 
 
 
 
9
  """
10
 
11
  from __future__ import annotations
12
 
 
13
  from typing import Any
14
 
15
- from src import payloads
16
  from src.endpoints._common import (
17
  Budget,
18
  Collector,
@@ -26,12 +37,25 @@ from src.endpoints._common import (
26
  stamped,
27
  )
28
  from src.identifiers import parse_gene_list
 
29
 
30
  ENDPOINT = "drugs_for_targets"
31
  OT_DRUGS = "OpenTargets_get_associated_drugs_by_target_ensemblID"
32
  CHEMBL_MECHANISMS = "ChEMBL_get_drug_mechanisms"
33
  MAX_MECHANISM_LOOKUPS = 30
34
  MAX_INDICATIONS = 10
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
 
37
  def drugs_for_targets(genes: str = "") -> dict[str, Any]:
@@ -126,27 +150,76 @@ def _attach_mechanisms(out: Collector, client, budget: Budget) -> int:
126
  for row in out.rows:
127
  key = (stage_rank(row["max_clinical_stage"]), row["drug"])
128
  best[row["drug_chembl_id"]] = min(best.get(row["drug_chembl_id"], key), key)
129
- order = sorted(best, key=lambda chembl_id: (best[chembl_id], chembl_id))
 
130
  mechanisms: dict[str, dict[str, Any]] = {}
131
- for chembl_id in order[:MAX_MECHANISM_LOOKUPS]:
132
- result = out.call(client, budget, CHEMBL_MECHANISMS, {"drug_chembl_id": chembl_id})
133
- if result is None:
134
- continue
135
- records = result.data.get("mechanisms") or []
136
- moa = sorted({m["mechanism_of_action"] for m in records if m.get("mechanism_of_action")})
137
- actions = sorted({m["action_type"] for m in records if m.get("action_type")})
138
- mechanisms[chembl_id] = stamped(
139
- {"mechanism_of_action": moa, "action_types": actions},
140
- result,
141
- ref=chembl_id,
142
- url=f"https://www.ebi.ac.uk/chembl/compound_report_card/{chembl_id}/",
143
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
  for row in out.rows:
145
  found = mechanisms.get(row["drug_chembl_id"])
146
  row["mechanisms"] = found if found is not None else {"looked_up": False}
147
  return len(mechanisms)
148
 
149
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
  def _narrative(out: Collector, per_target: list[tuple[str, int]], looked_up: int) -> str:
151
  sentences = []
152
  clauses = []
 
6
  advanced drugs in the call (stage first, then name); the payload says how many were looked up.
7
  Open Targets' coverage note travels with every payload: an investigational compound with no
8
  trial record can be absent (Phase 0: MRTX1133 against KRAS G12D).
9
+
10
+ The mechanism lookups share one ChEMBL window per call (``MECHANISM_WINDOW_SECONDS``) instead of
11
+ 30 s each, and a lookup ChEMBL does not answer is tried again inside it, round-robin with the
12
+ others still waiting. Why: over the 2026-10-29 → 11-05 smoke week, ChEMBL stopped answering
13
+ within 30 s right after 00, 06, 12 and 18 UTC (26 of 57 KRAS probes; the other sources answered
14
+ 98-100 % of the week), while the request itself is a 1.3 KB answer in under a second at other
15
+ times (timed by hand 2026-11-05). A slow
16
+ minute is the source's state, not one drug's, so waiting it out is the only thing that recovers
17
+ the answer. Whatever is still unanswered when the window closes is named once in
18
+ ``source_errors`` and its rows say ``looked_up: false``; the Open Targets rows always stand.
19
  """
20
 
21
  from __future__ import annotations
22
 
23
+ import time
24
  from typing import Any
25
 
26
+ from src import config, payloads
27
  from src.endpoints._common import (
28
  Budget,
29
  Collector,
 
37
  stamped,
38
  )
39
  from src.identifiers import parse_gene_list
40
+ from src.tu_client import SourceError
41
 
42
  ENDPOINT = "drugs_for_targets"
43
  OT_DRUGS = "OpenTargets_get_associated_drugs_by_target_ensemblID"
44
  CHEMBL_MECHANISMS = "ChEMBL_get_drug_mechanisms"
45
  MAX_MECHANISM_LOOKUPS = 30
46
  MAX_INDICATIONS = 10
47
+ # All of one call's ChEMBL mechanism lookups, retries included, finish inside this window (and
48
+ # inside what is left of the endpoint's own budget). Each single call still stops at 30 s.
49
+ MECHANISM_WINDOW_SECONDS = 150
50
+ # After a quick failure (an error answer, not silence) the next attempt waits this long, so a
51
+ # struggling source is not hammered. A timeout has already waited its 30 s and does not pause.
52
+ RETRY_PAUSE_SECONDS = 5
53
+ # A backstop for a lookup that keeps failing quickly; the window is the real bound.
54
+ MAX_ATTEMPTS_PER_LOOKUP = 5
55
+
56
+ # The mechanism window's clock and sleep, swapped for a fake clock in tests.
57
+ _clock = time.monotonic
58
+ _sleep = time.sleep
59
 
60
 
61
  def drugs_for_targets(genes: str = "") -> dict[str, Any]:
 
150
  for row in out.rows:
151
  key = (stage_rank(row["max_clinical_stage"]), row["drug"])
152
  best[row["drug_chembl_id"]] = min(best.get(row["drug_chembl_id"], key), key)
153
+ order = sorted(best, key=lambda chembl_id: (best[chembl_id], chembl_id))[:MAX_MECHANISM_LOOKUPS]
154
+ window = Budget(min(MECHANISM_WINDOW_SECONDS, budget.remaining), clock=_clock)
155
  mechanisms: dict[str, dict[str, Any]] = {}
156
+ tries = dict.fromkeys(order, 0)
157
+ last_failure: SourceError | None = None
158
+ pending = list(order)
159
+ while pending and not window.spent:
160
+ waiting = []
161
+ for chembl_id in pending:
162
+ if window.spent or tries[chembl_id] >= MAX_ATTEMPTS_PER_LOOKUP:
163
+ waiting.append(chembl_id)
164
+ continue
165
+ tries[chembl_id] += 1
166
+ started = _clock()
167
+ result, err = out.attempt(
168
+ client, window, CHEMBL_MECHANISMS, {"drug_chembl_id": chembl_id}
169
+ )
170
+ if err is not None:
171
+ if err.reason != "source_unavailable":
172
+ out.source_errors.append(err.as_payload()) # a drift is not retried
173
+ continue
174
+ last_failure = err
175
+ waiting.append(chembl_id)
176
+ if _clock() - started < RETRY_PAUSE_SECONDS and not window.spent:
177
+ _sleep(min(RETRY_PAUSE_SECONDS, window.remaining))
178
+ continue
179
+ records = result.data.get("mechanisms") or []
180
+ moa = sorted(
181
+ {m["mechanism_of_action"] for m in records if m.get("mechanism_of_action")}
182
+ )
183
+ actions = sorted({m["action_type"] for m in records if m.get("action_type")})
184
+ mechanisms[chembl_id] = stamped(
185
+ {"mechanism_of_action": moa, "action_types": actions},
186
+ result,
187
+ ref=chembl_id,
188
+ url=f"https://www.ebi.ac.uk/chembl/compound_report_card/{chembl_id}/",
189
+ )
190
+ pending = waiting
191
+ if all(tries[c] >= MAX_ATTEMPTS_PER_LOOKUP for c in pending):
192
+ break
193
+ if pending:
194
+ out.source_errors.append(_unanswered(pending, order, tries, last_failure, window))
195
  for row in out.rows:
196
  found = mechanisms.get(row["drug_chembl_id"])
197
  row["mechanisms"] = found if found is not None else {"looked_up": False}
198
  return len(mechanisms)
199
 
200
 
201
+ def _unanswered(
202
+ pending: list[str],
203
+ order: list[str],
204
+ tries: dict[str, int],
205
+ last_failure: SourceError | None,
206
+ window: Budget,
207
+ ) -> dict[str, str]:
208
+ """One ``source_errors`` entry for every mechanism lookup ChEMBL left unanswered."""
209
+ attempts = sum(tries.values())
210
+ how = (
211
+ last_failure.detail
212
+ if last_failure is not None
213
+ else f"not looked up: the {ENDPOINT} call budget ran out"
214
+ )
215
+ return SourceError(
216
+ config.source_of(CHEMBL_MECHANISMS),
217
+ "source_unavailable",
218
+ f"{how}; mechanisms not looked up for {len(pending)} of {len(order)} drugs "
219
+ f"({attempts} attempt{'s' if attempts != 1 else ''} in a {round(window.seconds):g} s window)",
220
+ ).as_payload()
221
+
222
+
223
  def _narrative(out: Collector, per_target: list[tuple[str, int]], looked_up: int) -> str:
224
  sentences = []
225
  clauses = []
src/tu_client.py CHANGED
@@ -290,7 +290,15 @@ class ToolClient:
290
  self._tu = build_tooluniverse()
291
  return self._tu.run_one_function({"name": tool, "arguments": args}, use_cache=False)
292
 
293
- def call(self, tool: str, args: dict[str, Any], *, max_wait: float = 10.0) -> SourceResult:
 
 
 
 
 
 
 
 
294
  if tool not in config.endpoint_tool_names():
295
  # A programming error, not a user input: endpoints only ever name loaded tools.
296
  raise ValueError(f"{tool!r} is not an endpoint tool")
@@ -309,18 +317,21 @@ class ToolClient:
309
  source, "rate_limited", f"{cfg['name']} limit of {cfg['rate_per_second']}/s reached"
310
  )
311
 
 
 
 
312
  runner = self._runner or self._default_runner
313
  future = self._pool.submit(runner, tool, dict(args))
314
  # `wait`, not `result(timeout=…)`: since Python 3.11 concurrent.futures.TimeoutError IS the
315
  # builtin TimeoutError, so a source's own read timeout raised inside the call would be
316
  # indistinguishable from our deadline passing (caught by a test on 2026-10-20).
317
- done, _ = concurrent.futures.wait([future], timeout=cfg["timeout_seconds"])
318
  if not done:
319
  future.cancel()
320
  raise SourceError(
321
  source,
322
  "source_unavailable",
323
- f"{cfg['name']} did not answer within {cfg['timeout_seconds']} s",
324
  )
325
  try:
326
  raw = future.result()
 
290
  self._tu = build_tooluniverse()
291
  return self._tu.run_one_function({"name": tool, "arguments": args}, use_cache=False)
292
 
293
+ def call(
294
+ self,
295
+ tool: str,
296
+ args: dict[str, Any],
297
+ *,
298
+ max_wait: float = 10.0,
299
+ deadline: float | None = None,
300
+ ) -> SourceResult:
301
+ """``deadline`` can shorten the source's ``timeout_seconds`` for this call, never extend it."""
302
  if tool not in config.endpoint_tool_names():
303
  # A programming error, not a user input: endpoints only ever name loaded tools.
304
  raise ValueError(f"{tool!r} is not an endpoint tool")
 
317
  source, "rate_limited", f"{cfg['name']} limit of {cfg['rate_per_second']}/s reached"
318
  )
319
 
320
+ timeout = cfg["timeout_seconds"]
321
+ if deadline is not None:
322
+ timeout = max(0.0, min(timeout, deadline))
323
  runner = self._runner or self._default_runner
324
  future = self._pool.submit(runner, tool, dict(args))
325
  # `wait`, not `result(timeout=…)`: since Python 3.11 concurrent.futures.TimeoutError IS the
326
  # builtin TimeoutError, so a source's own read timeout raised inside the call would be
327
  # indistinguishable from our deadline passing (caught by a test on 2026-10-20).
328
+ done, _ = concurrent.futures.wait([future], timeout=timeout)
329
  if not done:
330
  future.cancel()
331
  raise SourceError(
332
  source,
333
  "source_unavailable",
334
+ f"{cfg['name']} did not answer within {round(timeout, 1):g} s",
335
  )
336
  try:
337
  raw = future.result()
tests/test_chembl_window.py ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ChEMBL's slow window, offline: the KRAS probe's recording with ChEMBL made silent for a while.
2
+
3
+ The 2026-10-29 → 11-05 smoke week failed on one thing: right after 00, 06, 12 and 18 UTC ChEMBL did
4
+ not answer the KRAS probe's three mechanism lookups within 30 s (26 of 57 runs), and each lookup
5
+ waited its own 30 s, so a failing run took ~90 s and reported three identical errors. These tests
6
+ replay the recorded KRAS scenario (``q1_q2_drugs_kras``) on a fake clock, with ChEMBL silent for its
7
+ first N calls, and pin what the endpoint does now: it waits the silence out inside one shared
8
+ window, and when ChEMBL stays silent the Open Targets rows still stand, every row says its
9
+ mechanisms were not looked up, and ONE structured error names ChEMBL.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import time
15
+
16
+ import pytest
17
+
18
+ from scripts import smoke_live as sl
19
+ from src import config
20
+ from src.cache import ResponseCache
21
+ from src.endpoints import _common, drugs
22
+ from src.ratelimit import RateLimiters
23
+ from src.tu_client import SourceError, ToolClient
24
+ from tests.conftest import CassetteRunner, FakeClock, FakeRunner
25
+
26
+ CASSETTE = "q1_q2_drugs_kras"
27
+ DEADLINE = 30 # ChEMBL's timeout_seconds: what one silent call costs
28
+ KRAS_PROBE = next(p for p in sl.PROBES if p.id == "drugs_kras")
29
+
30
+
31
+ class SilentChembl:
32
+ """The recording, except ChEMBL's first ``silent`` calls fail the way production saw them."""
33
+
34
+ def __init__(self, clock: FakeClock, silent: int, *, quick: bool = False, drift: str = ""):
35
+ self.inner = CassetteRunner(CASSETTE)
36
+ self.clock = clock
37
+ self.silent = silent
38
+ self.quick = quick
39
+ self.drift = drift
40
+ self.chembl_calls: list[str] = []
41
+
42
+ def __call__(self, tool: str, args: dict):
43
+ if tool != drugs.CHEMBL_MECHANISMS:
44
+ return self.inner(tool, args)
45
+ self.chembl_calls.append(args["drug_chembl_id"])
46
+ if args["drug_chembl_id"] == self.drift:
47
+ return {"status": "success", "data": {"mechanism": []}} # no `mechanisms` key
48
+ if self.silent > 0:
49
+ self.silent -= 1
50
+ if self.quick:
51
+ raise ConnectionError("connection reset by peer")
52
+ self.clock.advance(DEADLINE)
53
+ # What the client raises when its deadline passes (src/tu_client.py).
54
+ raise SourceError("chembl", "source_unavailable", "ChEMBL did not answer within 30 s")
55
+ return self.inner(tool, args)
56
+
57
+
58
+ @pytest.fixture()
59
+ def kras(monkeypatch):
60
+ """``kras(silent, ...)`` → (payload, runner, seconds of fake time the call took)."""
61
+
62
+ def run(silent: int, **kwargs):
63
+ meta = CassetteRunner(CASSETTE).cassette["_recorded"]
64
+ clock = FakeClock(start=meta["recorded_at_epoch"])
65
+ runner = SilentChembl(clock, silent, **kwargs)
66
+ monkeypatch.setattr(drugs, "_clock", clock)
67
+ monkeypatch.setattr(drugs, "_sleep", clock.sleep)
68
+ _common.set_client(
69
+ ToolClient(
70
+ runner=runner,
71
+ cache=ResponseCache(":memory:", clock=clock),
72
+ limiters=RateLimiters(clock=clock, sleep=clock.sleep),
73
+ clock=clock,
74
+ tooluniverse_version=meta["tooluniverse_version"],
75
+ )
76
+ )
77
+ started = clock.now
78
+ payload = drugs.drugs_for_targets(*meta["arguments"])
79
+ return payload, runner, clock.now - started, clock
80
+
81
+ yield run
82
+ _common.set_client(None)
83
+
84
+
85
+ def _looked_up(payload) -> list[bool]:
86
+ return [r["mechanisms"].get("looked_up", True) for r in payload["rows"]]
87
+
88
+
89
+ def test_the_recording_takes_one_call_per_drug(kras):
90
+ payload, runner, seconds, _ = kras(silent=0)
91
+ assert payload["source_errors"] == []
92
+ assert len(runner.chembl_calls) == 3 and seconds == 0
93
+ assert sl.evaluate(KRAS_PROBE, payload)[1] is None
94
+
95
+
96
+ def test_a_slow_chembl_window_is_waited_out(kras):
97
+ """Silent for all but the window's last call: before this fix, three errors and no mechanisms."""
98
+ silent = drugs.MECHANISM_WINDOW_SECONDS // DEADLINE - 1
99
+ payload, runner, seconds, _ = kras(silent=silent)
100
+ assert payload["status"] == "ok"
101
+ assert payload["source_errors"] == []
102
+ assert _looked_up(payload) == [True, True, True]
103
+ assert payload["mechanism_lookups"]["looked_up"] == 3
104
+ assert len(runner.chembl_calls) == silent + 3
105
+ assert seconds <= drugs.MECHANISM_WINDOW_SECONDS
106
+ assert sl.evaluate(KRAS_PROBE, payload)[1] is None # the smoke probe passes
107
+
108
+
109
+ def test_silent_lookups_are_retried_round_robin(kras, monkeypatch):
110
+ """A slow minute is ChEMBL's state, not one drug's: each pending drug is tried in turn."""
111
+ monkeypatch.setattr(drugs, "MECHANISM_WINDOW_SECONDS", 150) # room for five silent calls
112
+ _, runner, _, _ = kras(silent=4)
113
+ first_pass = runner.chembl_calls[:3]
114
+ assert sorted(first_pass) == sorted(set(first_pass)) # three different drugs first
115
+ assert runner.chembl_calls[3:5] == first_pass[:2] # then the first two again, in order
116
+
117
+
118
+ def test_when_chembl_stays_silent_the_rows_stand_and_one_error_names_it(kras):
119
+ payload, runner, seconds, _ = kras(silent=10_000)
120
+ assert payload["status"] == "ok"
121
+ assert [r["drug"] for r in payload["rows"]] == ["ADAGRASIB", "SOTORASIB", "SALIRASIB"]
122
+ assert _looked_up(payload) == [False, False, False]
123
+ attempts = drugs.MECHANISM_WINDOW_SECONDS // DEADLINE
124
+ assert payload["source_errors"] == [
125
+ {
126
+ "source": "chembl",
127
+ "reason": "source_unavailable",
128
+ "detail": (
129
+ "ChEMBL did not answer within 30 s; mechanisms not looked up for 3 of 3 drugs "
130
+ f"({attempts} attempt{'s' if attempts != 1 else ''} in a "
131
+ f"{drugs.MECHANISM_WINDOW_SECONDS} s window)"
132
+ ),
133
+ }
134
+ ]
135
+ assert len(runner.chembl_calls) == attempts
136
+ assert seconds == drugs.MECHANISM_WINDOW_SECONDS
137
+ assert payload["narrative"].count("chembl source_unavailable") == 1
138
+ facts, failure = sl.evaluate(KRAS_PROBE, payload)
139
+ # Still a failed probe, but a structured one: the smoke measure's C4 holds.
140
+ assert failure["class"] == "source_unavailable" and failure["source"] == "chembl"
141
+ assert facts["n_rows"] == 3
142
+
143
+
144
+ def test_a_quick_failure_pauses_before_the_next_attempt(kras):
145
+ payload, runner, _, clock = kras(silent=2, quick=True)
146
+ assert payload["source_errors"] == []
147
+ assert _looked_up(payload) == [True, True, True]
148
+ assert clock.sleeps == [drugs.RETRY_PAUSE_SECONDS] * 2
149
+ assert len(runner.chembl_calls) == 5
150
+
151
+
152
+ def test_a_drifted_mechanism_answer_is_named_and_not_retried(kras):
153
+ payload, runner, _, _ = kras(silent=0, drift="CHEMBL23293")
154
+ assert runner.chembl_calls.count("CHEMBL23293") == 1
155
+ assert [e["reason"] for e in payload["source_errors"]] == ["upstream_schema_drift"]
156
+ assert sorted(_looked_up(payload)) == [False, True, True]
157
+
158
+
159
+ def test_the_window_never_outlasts_what_is_left_of_the_endpoint_budget(monkeypatch):
160
+ monkeypatch.setattr(drugs, "MECHANISM_WINDOW_SECONDS", 150) # longer than the 40 s left
161
+ clock = FakeClock()
162
+ monkeypatch.setattr(drugs, "_clock", clock)
163
+ monkeypatch.setattr(drugs, "_sleep", clock.sleep)
164
+ deadlines: list[float] = []
165
+
166
+ class Client:
167
+ def call(self, tool, args, *, max_wait, deadline):
168
+ deadlines.append(deadline)
169
+ waited = min(DEADLINE, deadline) # the real client caps it at timeout_seconds
170
+ clock.advance(waited)
171
+ raise SourceError(
172
+ "chembl", "source_unavailable", f"ChEMBL did not answer within {waited:g} s"
173
+ )
174
+
175
+ out = _common.Collector()
176
+ out.rows = [
177
+ {"drug": name, "drug_chembl_id": cid, "max_clinical_stage": "PHASE_2"}
178
+ for name, cid in (("A", "CHEMBL1"), ("B", "CHEMBL2"), ("C", "CHEMBL3"))
179
+ ]
180
+ assert drugs._attach_mechanisms(out, Client(), _common.Budget(40, clock=clock)) == 0
181
+ assert deadlines == [40, 10] # the second call gets only what is left
182
+ assert clock.now - FakeClock().now == 40
183
+ [error] = out.source_errors
184
+ assert error["detail"] == (
185
+ "ChEMBL did not answer within 10 s; mechanisms not looked up for 3 of 3 drugs "
186
+ "(2 attempts in a 40 s window)"
187
+ )
188
+ assert [r["mechanisms"] for r in out.rows] == [{"looked_up": False}] * 3
189
+
190
+
191
+ def test_a_caller_deadline_shortens_but_never_extends_the_source_timeout(
192
+ monkeypatch, memory_cache, clock
193
+ ):
194
+ def hangs(args):
195
+ time.sleep(1)
196
+
197
+ client = ToolClient(
198
+ runner=FakeRunner({drugs.CHEMBL_MECHANISMS: hangs}),
199
+ cache=memory_cache,
200
+ clock=clock,
201
+ tooluniverse_version="1.5.6",
202
+ )
203
+ started = time.monotonic()
204
+ with pytest.raises(SourceError) as shortened:
205
+ client.call(drugs.CHEMBL_MECHANISMS, {"drug_chembl_id": "CHEMBL1"}, deadline=0.2)
206
+ assert time.monotonic() - started < 0.9
207
+ assert shortened.value.detail == "ChEMBL did not answer within 0.2 s"
208
+
209
+ monkeypatch.setitem(config.source("chembl"), "timeout_seconds", 0.2)
210
+ started = time.monotonic()
211
+ with pytest.raises(SourceError) as capped:
212
+ client.call(drugs.CHEMBL_MECHANISMS, {"drug_chembl_id": "CHEMBL2"}, deadline=60)
213
+ assert time.monotonic() - started < 0.9
214
+ assert capped.value.detail == "ChEMBL did not answer within 0.2 s"