feat(trials): filters, polish & eligibility on main (recover #29 + #30) (#32)

#28 (autocomplete) reached main, but stacked PRs #29 (filters + polish) and #30
(eligibility) merged into their base branches, not up to main. This lands their
content on main so the full Clinical Trials tab work ships: study-type/status
filters + defaults, layout & number-consistency polish, and enrollment/eligibility
(ingestion capture, backfill script, active-trial display).

Data change: run scripts/upload_data.py --only trials before deploying.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -13,11 +13,11 @@ pinned: false
 
 **ALS Research Intelligence for Physicians**
 
-Candle-fire is a physician-facing tool that synthesizes evidence from ~500 curated ALS research papers and a biomedical knowledge graph. Ask a free-text question about ALS biology, drug targets, or clinical trials — get a structured, cited answer in under 30 seconds.
+Candle-fire is a physician-facing tool that synthesizes evidence from ~10,000 curated ALS research papers and a biomedical knowledge graph. Ask a free-text question about ALS biology, drug targets, or clinical trials — get a structured, cited answer in under 30 seconds.
 
 ## What It Does
 
-- **Two-layer retrieval**: Knowledge graph expansion (BioLORD-2023-C embeddings + NetworkX) → RAG over ~500 ALS papers
+- **Two-layer retrieval**: Knowledge graph expansion (BioLORD-2023-C embeddings + NetworkX) → RAG over ~10,000 ALS papers
 - **Citation-weighted ranking**: Highly-cited papers surface first
 - **Structured synthesis**: Claude Sonnet produces mechanism summaries, entity tables, evidence strength assessments, and trial links
 - **Biomedical synonyms**: BioLORD understands that "TDP-43" = "TARDBP" = "TAR DNA-binding protein 43"
@@ -47,7 +47,7 @@ cp .env.example .env
 Build the knowledge assets before launching the app. Each step is resumable.
 
 ```bash
-# 1. Ingest ~500 ALS papers from PubMed + PMC full text + citation counts (~15 min)
+# 1. Ingest ~10,000 ALS papers from PubMed + PMC full text + citation counts (~15 min)
 uv run python scripts/ingest_papers.py
 
 # 2. Ingest ALS clinical trials from ClinicalTrials.gov (< 1 min, run in parallel)
@@ -131,10 +131,10 @@ Physician query
 
 | Source | Content | Volume |
 |---|---|---|
-| PubMed Entrez | ALS paper abstracts + metadata | ~500 papers (2018–2024) |
+| PubMed Entrez | ALS paper abstracts + metadata | ~10,000 papers |
 | PubMed Central | Full text for Open Access papers | ~50% coverage |
 | Semantic Scholar | Citation counts per paper | All papers |
-| ClinicalTrials.gov v2 | Active ALS recruiting trials | ~112 trials |
+| ClinicalTrials.gov v2 | Interventional + expanded-access ALS trials | ~720 trials |
 
 ## Disclaimer
 

app.py

@@ -1,6 +1,7 @@
 """Gradio web UI for candle-fire — physician-facing ALS research intelligence."""
 from __future__ import annotations
 
+import html
 import json
 from pathlib import Path
 
@@ -103,7 +104,13 @@ _trials = _load_trials()
 _client = anthropic.Anthropic()
 
 _n_chunks = _collection.count() if _collection else 0
-_n_trials = len(_trials)
+# Headline trial count = recruiting, interventional trials only (the actionable set), not the
+# full corpus (which includes completed/terminated studies and expanded-access programs).
+_RECRUITING = {"RECRUITING", "NOT_YET_RECRUITING", "ENROLLING_BY_INVITATION", "AVAILABLE"}
+_n_trials = sum(
+    1 for t in _trials
+    if t.get("study_type") == "INTERVENTIONAL" and t.get("status") in _RECRUITING
+)
 _kg_nodes = _graph.number_of_nodes() if _graph else 0
 
 # Experimental therapy landscape (offline-built artifact; loaded once)
@@ -171,12 +178,15 @@ footer { display: none !important; }
 /* Autocomplete gate: hide a combobox's attached option list until ≥3 chars (see
    _AUTOCOMPLETE_GATE_JS). The script toggles .ac-hide on the input's wrapper by length. */
 #facility_combo.ac-hide ul, #city_combo.ac-hide ul { display: none !important; }
+/* Smaller filter labels so long ones (e.g. "Recruitment status") stay on one line and the
+   dropdown chevron doesn't overlap the text. */
+.trial-filters label span { font-size: 0.78rem !important; white-space: nowrap; }
 """
 
 _TITLE_MD = """# 🕯️ Candle-Fire
 ### ALS Research Intelligence for Physicians
 Ask a free-text question about ALS biology, drug targets, or clinical trials.
-Answers are synthesized from ~500 curated ALS papers and enriched by a biomedical knowledge graph.
+Answers are synthesized from a curated ALS research corpus and enriched by a biomedical knowledge graph.
 """
 
 _DISCLAIMER_MD = """<div class="disclaimer">
@@ -223,23 +233,24 @@ _LOC_CHOICES = trials_query.location_choices(_LOC_INDEX)
 _AUTOCOMPLETE_GATE_JS = f"""
 () => {{
   const MIN = {trials_query.MIN_AUTOCOMPLETE_CHARS};
-  const gate = (id) => {{
+  const gate = (id, hint) => {{
     const root = document.getElementById(id);
     if (!root) return;
     const input = root.querySelector('input');
     if (!input) return;
+    if (hint) input.setAttribute('placeholder', hint);  // in-box hint; hides once they type
     const apply = () => root.classList.toggle('ac-hide', input.value.trim().length < MIN);
     input.addEventListener('input', apply);
     input.addEventListener('focus', apply);
     apply();
   }};
-  gate('facility_combo');
-  gate('city_combo');
+  gate('facility_combo', 'Type ≥3 letters, e.g. Mass General');
+  gate('city_combo', 'Type ≥3 letters, busiest cities first');
 }}
 """
 
 
-def _search_trials(facility: str, state: str, city: str, status: str) -> str:
+def _search_trials(facility: str, state: str, city: str, study_type: str, status: str) -> str:
     facility = (facility or "").strip() or None
     city = (city or "").strip() or None
     state = None if (not state or state == "All") else state
@@ -248,8 +259,28 @@ def _search_trials(facility: str, state: str, city: str, status: str) -> str:
         return '<div style="color:#888;padding:12px 0;">Enter a facility, state, or city to search.</div>'
 
     matches = trials_query.search_trials_by_location(
-        _trials, facility=facility, city=city, state=state, status=status
+        _trials, facility=facility, city=city, state=state,
+        status=status, study_type=study_type,
     )
+
+    # If the active filters hide everything, say whether broader filters would find trials —
+    # e.g. a facility with only completed studies under the default Recruiting + Interventional.
+    if not matches and (status != "All" or study_type != "All"):
+        broad = trials_query.search_trials_by_location(
+            _trials, facility=facility, city=city, state=state, status="All", study_type="All",
+        )
+        if broad:
+            where = ", ".join(p for p in (facility, city, state) if p)
+            return (
+                '<div style="background:#fff6e5;border:1px solid #ffe0a3;border-radius:8px;'
+                'padding:10px 12px;margin:6px 0;color:#7a5b00;font-size:0.9rem;">'
+                f'No <b>{html.escape((study_type or "").lower())}</b> trials that are '
+                f'<b>{html.escape((status or "").lower())}</b> at {html.escape(where)}. '
+                f'{len(broad)} trial(s) exist there under broader filters — set '
+                '<b>Study type</b> and <b>Recruitment status</b> to <b>All</b> to see them.'
+                '</div>'
+            )
+
     enriched = [
         trials_query.enrich_trial(t, _collection, _graph, _trials)
         for t in matches[:_TRIAL_ENRICH_CAP]
@@ -269,7 +300,7 @@ with gr.Blocks(title="Candle-Fire — ALS Research Intelligence") as demo:
                 gr.HTML(
                     f'<div class="status-bar">'
                     f'{_n_chunks} paper chunks &nbsp;·&nbsp; '
-                    f'{_n_trials} clinical trials &nbsp;·&nbsp; '
+                    f'{_n_trials} recruiting interventional trials &nbsp;·&nbsp; '
                     f'{_kg_nodes} knowledge graph nodes'
                     f'</div>'
                 )
@@ -380,12 +411,11 @@ with gr.Blocks(title="Candle-Fire — ALS Research Intelligence") as demo:
                     "**location** (state / city). Each result is enriched with recruiting status, an "
                     "evidence-strength tier, key supporting papers, and related trials for the same compound."
                 )
-                with gr.Row():
+                with gr.Row(elem_classes="trial-filters"):
                     facility_tb = gr.Dropdown(
                         choices=_LOC_CHOICES["facilities"], value=None,
                         label="Facility / institution", scale=2,
                         filterable=True, allow_custom_value=True, elem_id="facility_combo",
-                        info="Type ≥3 letters and pick a match (e.g. Mass General).",
                     )
                     state_dd = gr.Dropdown(
                         choices=_US_STATES, value="All", label="State", scale=1,
@@ -394,11 +424,17 @@ with gr.Blocks(title="Candle-Fire — ALS Research Intelligence") as demo:
                         choices=_LOC_CHOICES["cities"], value=None,
                         label="City", scale=1,
                         filterable=True, allow_custom_value=True, elem_id="city_combo",
-                        info="Type ≥3 letters; busiest trial cities first.",
+                    )
+                # Filters on their own row so the labels/values have full width — no wrapping,
+                # no value running under the chevron.
+                with gr.Row(elem_classes="trial-filters"):
+                    study_type_dd = gr.Dropdown(
+                        choices=["Interventional", "Expanded Access", "All"],
+                        value="Interventional", label="Study type", scale=1,
                     )
                     trial_status_dd = gr.Dropdown(
                         choices=["All", "Recruiting", "Not recruiting"],
-                        value="All", label="Recruitment status", scale=1,
+                        value="Recruiting", label="Recruitment status", scale=1,
                     )
                 search_btn = gr.Button("Search trials", variant="primary")
                 trial_results = gr.HTML(
@@ -408,7 +444,7 @@ with gr.Blocks(title="Candle-Fire — ALS Research Intelligence") as demo:
                 # Facility/city are typeable comboboxes (filterable Dropdowns) — the physician
                 # types and picks from the attached, busiest-first list. No per-keystroke server
                 # event needed; the search reads the selected/typed value directly.
-                _trial_search_inputs = [facility_tb, state_dd, city_tb, trial_status_dd]
+                _trial_search_inputs = [facility_tb, state_dd, city_tb, study_type_dd, trial_status_dd]
                 search_btn.click(_search_trials, inputs=_trial_search_inputs, outputs=[trial_results])
                 # Picking a facility/city from its list also runs the search immediately.
                 facility_tb.select(_search_trials, inputs=_trial_search_inputs, outputs=[trial_results])

ingestion/clinicaltrials.py

@@ -70,6 +70,22 @@ def fetch_als_trials(
     return trials
 
 
+def _extract_eligibility(elig_mod: dict) -> dict:
+    """Enrollment/eligibility fields from a CT.gov v2 eligibilityModule.
+
+    Shared by _flatten_trial (ingestion) and scripts/backfill_eligibility.py so the stored
+    shape is identical whichever path populated it.
+    """
+    return {
+        "criteria": elig_mod.get("eligibilityCriteria", ""),  # free text: Inclusion/Exclusion
+        "sex": elig_mod.get("sex", ""),                        # ALL / MALE / FEMALE
+        "min_age": elig_mod.get("minimumAge", ""),             # e.g. "18 Years"
+        "max_age": elig_mod.get("maximumAge", ""),
+        "healthy_volunteers": elig_mod.get("healthyVolunteers"),
+        "std_ages": elig_mod.get("stdAges", []),               # e.g. ["ADULT", "OLDER_ADULT"]
+    }
+
+
 def _flatten_trial(study: dict) -> dict:
     proto = study.get("protocolSection", {})
     id_mod = proto.get("identificationModule", {})
@@ -79,6 +95,7 @@ def _flatten_trial(study: dict) -> dict:
     arms_mod = proto.get("armsInterventionsModule", {})
     status_mod = proto.get("statusModule", {})
     contacts_mod = proto.get("contactsLocationsModule", {})
+    elig_mod = proto.get("eligibilityModule", {})
 
     nct_id = id_mod.get("nctId", "")
     interventions = [
@@ -126,6 +143,7 @@ def _flatten_trial(study: dict) -> dict:
         "locations": locations,
         "contact_phone": contact_phone,
         "contact_email": contact_email,
+        "eligibility": _extract_eligibility(elig_mod),
     }
 
 

models.py

@@ -118,6 +118,9 @@ class TrialSummary:
     locations: list[dict] = field(default_factory=list)
     contact_phone: str = ""
     contact_email: str = ""
+    # Enrollment/eligibility from CT.gov: {criteria, sex, min_age, max_age,
+    # healthy_volunteers, std_ages}. Shown for active/recruiting trials.
+    eligibility: dict = field(default_factory=dict)
 
 
 @dataclass

scripts/backfill_eligibility.py

@@ -0,0 +1,104 @@
+"""Backfill enrollment/eligibility criteria into an existing trials.jsonl.
+
+Adds the `eligibility` field to trials that don't have it by fetching each trial's
+eligibilityModule from ClinicalTrials.gov v2 — WITHOUT re-running the expensive LLM target
+extraction that a full `ingest_trials.py` would. Fetches in batches via `filter.ids`, merges
+in place, and rewrites trials.jsonl. Idempotent: re-running only fills trials still missing it.
+
+Usage:
+  uv run python scripts/backfill_eligibility.py            # backfill missing eligibility
+  uv run python scripts/backfill_eligibility.py --force    # refetch for ALL trials
+  uv run python scripts/backfill_eligibility.py --dry-run  # report only, no write
+
+After it writes trials.jsonl, deploy per the runbook:
+  uv run python scripts/upload_data.py --only trials   # push to the HF dataset repo
+  git push hf main
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+import httpx
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from config import CTGOV_BASE, TRIALS_PATH  # noqa: E402
+from ingestion.clinicaltrials import _extract_eligibility  # noqa: E402
+
+_BATCH = 50          # NCT ids per request (filter.ids)
+_PAUSE_S = 0.34      # ~3 req/s, polite to CT.gov
+
+
+def _fetch_eligibility(nct_ids: list[str]) -> dict[str, dict]:
+    """{nct_id: eligibility dict} for a batch of NCT ids."""
+    resp = httpx.get(
+        CTGOV_BASE,
+        params={
+            "filter.ids": ",".join(nct_ids),
+            "fields": "NCTId,EligibilityModule",
+            "pageSize": len(nct_ids),
+        },
+        timeout=30,
+    )
+    resp.raise_for_status()
+    out: dict[str, dict] = {}
+    for study in resp.json().get("studies", []):
+        proto = study.get("protocolSection", {})
+        nct = proto.get("identificationModule", {}).get("nctId", "")
+        if nct:
+            out[nct] = _extract_eligibility(proto.get("eligibilityModule", {}))
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Backfill eligibility criteria into trials.jsonl.")
+    ap.add_argument("--force", action="store_true", help="Refetch for all trials, not just missing.")
+    ap.add_argument("--dry-run", action="store_true", help="Report what would change; write nothing.")
+    args = ap.parse_args()
+
+    if not TRIALS_PATH.exists():
+        print(f"ERROR — {TRIALS_PATH} not found (run ingest_trials.py first).", file=sys.stderr)
+        return 1
+
+    trials = [json.loads(line) for line in TRIALS_PATH.read_text().splitlines() if line.strip()]
+    todo = [
+        t for t in trials
+        if t.get("nct_id") and (args.force or not (t.get("eligibility") or {}).get("criteria"))
+    ]
+    print(f"{len(trials)} trials; {len(todo)} to fetch eligibility for"
+          f"{' (force)' if args.force else ''}.")
+    if not todo:
+        print("Nothing to do.")
+        return 0
+    if args.dry_run:
+        print("--dry-run: no fetch, no write.")
+        return 0
+
+    by_nct = {t["nct_id"]: t for t in trials}
+    fetched = 0
+    for i in range(0, len(todo), _BATCH):
+        ids = [t["nct_id"] for t in todo[i : i + _BATCH]]
+        try:
+            for nct, elig in _fetch_eligibility(ids).items():
+                if nct in by_nct:
+                    by_nct[nct]["eligibility"] = elig
+                    fetched += 1
+        except httpx.HTTPError as exc:
+            print(f"  batch {i // _BATCH} failed: {exc}", file=sys.stderr)
+        print(f"  {min(i + _BATCH, len(todo))}/{len(todo)}")
+        if i + _BATCH < len(todo):
+            time.sleep(_PAUSE_S)
+
+    with_crit = sum(1 for t in trials if (t.get("eligibility") or {}).get("criteria"))
+    TRIALS_PATH.write_text("".join(json.dumps(t, ensure_ascii=False) + "\n" for t in trials))
+    print(f"Fetched {fetched}; {with_crit}/{len(trials)} trials now have eligibility criteria.")
+    print(f"Wrote {TRIALS_PATH}. Next: upload_data.py --only trials, then git push hf main.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

trials_query.py

@@ -112,19 +112,29 @@ def search_trials_by_location(
     state: str | None = None,
     country: str | None = None,
     status: str | None = None,
+    study_type: str | None = None,
 ) -> list[dict]:
     """Return trials with at least one site matching the location filters.
 
     `status`: "Recruiting" keeps only trials with an open overall status; "Not recruiting"
-    keeps only closed ones; anything else (None / "All") keeps all. Each returned trial is a
-    shallow copy with `matched_sites` attached; recruiting trials are ranked first.
+    keeps only closed ones; anything else (None / "All") keeps all.
+    `study_type`: "Interventional" keeps interventional trials; "Expanded Access" keeps
+    expanded-access (investigational-use) programs; anything else (None / "All") keeps both.
+    Each returned trial is a shallow copy with `matched_sites` attached; recruiting first.
     """
     if not any([facility, city, state, country]):
         return []
 
     status_filter = (status or "").strip().lower()
+    type_filter = (study_type or "").strip().lower()
     results: list[dict] = []
     for trial in trials:
+        is_eap = bool(trial.get("is_expanded_access"))
+        if type_filter == "interventional" and is_eap:
+            continue
+        if type_filter == "expanded access" and not is_eap:
+            continue
+
         matched_sites = [
             s for s in trial.get("locations", [])
             if _site_matches(s, facility, city, state, country)
@@ -158,22 +168,31 @@ MIN_AUTOCOMPLETE_CHARS = 3  # client-side gate: the attached list stays hidden u
 
 
 def build_location_index(trials: list[dict]) -> dict[str, list[tuple[str, int]]]:
-    """Distinct facility + city names with their trial-site counts, each sorted busiest first.
+    """Distinct facility + city names with their TRIAL counts, each sorted busiest first.
 
-    Built once at startup; feeds location_choices() which becomes the combobox `choices`.
+    Counts distinct trials, not site rows: a single trial that lists an anonymized placeholder
+    like "GSK Investigational Site" 49 times counts once, so placeholders don't balloon to the
+    top of the ranking and the shown count matches what a search can return. Built once at
+    startup; feeds location_choices() which becomes the combobox `choices`.
     """
     from collections import Counter
 
     city_counts: Counter = Counter()
     facility_counts: Counter = Counter()
     for t in trials:
+        cities_here: set[str] = set()
+        facilities_here: set[str] = set()
         for s in t.get("locations", []):
             city = (s.get("city") or "").strip()
             facility = (s.get("facility") or "").strip()
             if city:
-                city_counts[city] += 1
+                cities_here.add(city)
             if facility:
-                facility_counts[facility] += 1
+                facilities_here.add(facility)
+        for c in cities_here:
+            city_counts[c] += 1
+        for f in facilities_here:
+            facility_counts[f] += 1
 
     def _ranked(counter: Counter) -> list[tuple[str, int]]:
         # busiest first, then alphabetical for stable ties
@@ -182,16 +201,17 @@ def build_location_index(trials: list[dict]) -> dict[str, list[tuple[str, int]]]
     return {"cities": _ranked(city_counts), "facilities": _ranked(facility_counts)}
 
 
-def location_choices(index: dict) -> dict[str, list[tuple[str, str]]]:
-    """Gradio combobox (label, value) choices for cities and facilities, busiest first.
+def location_choices(index: dict) -> dict[str, list[str]]:
+    """Gradio combobox choices — plain names, busiest first.
 
-    Label carries the site count ("New York  ·  100 sites"); value is the clean name used for
-    searching. Order is preserved by Gradio's client-side filter, so ranking holds as you type.
+    Just the names (no "· N trials" suffix): with a filterable/custom-value Dropdown the
+    displayed option text becomes the field value, so any suffix would leak into the search
+    term. Ranking is preserved by list order; the count is used only for that ordering.
     """
-    def _fmt(pairs: list[tuple[str, int]]) -> list[tuple[str, str]]:
-        return [(f"{name}  ·  {c} site{'s' if c != 1 else ''}", name) for name, c in pairs]
-
-    return {"cities": _fmt(index.get("cities", [])), "facilities": _fmt(index.get("facilities", []))}
+    return {
+        "cities": [name for name, _ in index.get("cities", [])],
+        "facilities": [name for name, _ in index.get("facilities", [])],
+    }
 
 
 def _find_supporting_papers(
@@ -433,6 +453,7 @@ def enrich_trial(
         "url": trial.get("url", ""),
         "target_entities": trial.get("target_entities", []),
         "mechanism": mechanism,
+        "eligibility": trial.get("eligibility", {}) or {},
         "matched_sites": trial.get("matched_sites", []),
         "key_papers": key_papers,
         "sibling_trials": sibling_trials,
@@ -489,6 +510,36 @@ def _tier_rationale_html(ev: dict) -> str:
     )
 
 
+def _eligibility_html(elig: dict) -> str:
+    """Collapsible enrollment-criteria block: age/sex summary + inclusion/exclusion text.
+
+    Rendered only for active/recruiting trials (the caller gates on status), since that's when
+    a physician assesses whether a patient qualifies.
+    """
+    criteria = (elig.get("criteria") or "").strip()
+    if not criteria:
+        return ""
+    bits = []
+    age = " – ".join(x for x in (elig.get("min_age"), elig.get("max_age")) if x) or None
+    if age:
+        bits.append(f"Age {html.escape(age)}")
+    sex = elig.get("sex")
+    if sex and sex != "ALL":
+        bits.append(html.escape(sex.title()))
+    elif sex == "ALL":
+        bits.append("All sexes")
+    if elig.get("healthy_volunteers"):
+        bits.append("Accepts healthy volunteers")
+    summary = "Eligibility" + (f" · {' · '.join(bits)}" if bits else "")
+    body = html.escape(criteria).replace("\n", "<br>")
+    return (
+        '<details style="margin-top:6px;font-size:0.82rem;color:#555;">'
+        f'<summary style="cursor:pointer;color:#0984E3;">{summary}</summary>'
+        f'<div style="margin:4px 0 0 4px;line-height:1.4;">{body}</div>'
+        '</details>'
+    )
+
+
 def render_trials_html(enriched: list[dict], match_count: int) -> str:
     """Render enriched location-search results as an HTML card list."""
     if not enriched:
@@ -505,6 +556,8 @@ def render_trials_html(enriched: list[dict], match_count: int) -> str:
         mech = t.get("mechanism", "")
         mech_pill = _pill(f'Mechanism: {mech}', "#6C5CE7") if mech else ""
         rationale_html = _tier_rationale_html(ev)
+        # Enrollment criteria only for active/recruiting trials — the enrollable ones.
+        elig_html = _eligibility_html(t.get("eligibility", {})) if t.get("is_recruiting") else ""
         phase = html.escape((t.get("phase") or "—").replace("PHASE", "Ph"))
         title = html.escape(t.get("title", "")[:140])
         nct = html.escape(t.get("nct_id", ""))
@@ -553,7 +606,7 @@ def render_trials_html(enriched: list[dict], match_count: int) -> str:
             f'<div style="font-weight:600;">{nct_link} — {title}</div>'
             f'{rationale_html}'
             f'<div style="margin-top:4px;font-size:0.82rem;color:#555;"><b>Site(s):</b><br>{sites_html}</div>'
-            f'{papers_html}{siblings_html}'
+            f'{elig_html}{papers_html}{siblings_html}'
             '</div>'
         )