deploy API

serving/app/main.py

@@ -34,21 +34,28 @@ from typing import Literal
 
 import numpy as np
 from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
-from fastapi.middleware.cors import CORSMiddleware
 from fastapi.concurrency import run_in_threadpool
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse, JSONResponse
 from PIL import Image, ImageOps, UnidentifiedImageError
 from pydantic import BaseModel, Field, ValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
 
+from epills.core.catalogue.confidence import assess
+from epills.core.catalogue.store import IMPRINT_ALPHA, IMPRINT_TOP_K
 from epills.core.imprint.match import GALLERY_MIN_WEIGHT, read_hits
 from epills.core.inference.matching import check_imprint, decide
+from epills.core.inference.pill_finder import detect_pills
 from epills.core.inference.preprocess import WINDOW, crop_window
 from epills.core.safety import openfda
 from epills.core.safety.db import SafetyDB
-from epills.core.safety.rules import check_dose, check_medications, resolve_medication
+from epills.core.safety.rules import SOURCE_SCHEDULE, Alert, check_dose, check_medications, resolve_medication
 from epills.core.safety.rxnorm import RxNormClient, RxNormError
+from serving.app import schemas as S
 
-API_VERSION = "0.5.0"
+API_VERSION = "0.6.0"
+STARTED_AT = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
 LOCAL = {"safety_db": Path("data/processed/safety/safety.sqlite"),
          "embedder": Path("outputs/serving/stage2_model.pt"),
          "catalogue": Path("data/processed/catalogue")}
@@ -56,11 +63,18 @@ IDENTIFY_NOTE = ("Possible matches from a catalogue of US pills, ranked by how a
                  "what a pill is. Check with your pharmacist.")
 LABEL_NOTE = ("Medicines read from the photo. Check the name and strength against the box or label before adding "
               "it to your list.")
-MATCH_MESSAGES = {
-    "match": "This is your {name}.",
-    "unknown": "This pill isn't one of your medicines. Set it aside. Check with your pharmacist.",
+VISUAL_LIMITS = ("A visual comparison with reference photos only: it can't confirm what the medicine is, its strength, "
+                 "or that it is safe to take.")
+NO_PILL = "No pill found. Spread the pills apart on a plain background and photograph them again."
+STAND_IN = "stand-in finder (plain backgrounds only)"
+MATCH_MESSAGES = {  # visual evidence only (blueprint D5): "looks like", never "is"
+    "match": "This pill looks like your {name}.",
+    "unknown": "This pill doesn't look like any of your medicines. Set it aside. Check with your pharmacist.",
     "ambiguous": "This pill looks like more than one of your medicines. Turn it over and photograph it again.",
 }
+ERROR_CODES = {401: "unauthorised", 404: "not_found", 409: "version_mismatch", 422: "invalid_input",
+               502: "upstream_failed", 503: "not_ready"}
+RETRY_AFTER = 20  # seconds, with 503 not_ready (a part still loading, or the Space waking up)
 
 
 def _asset(env, repo_path, local):
@@ -163,7 +177,9 @@ app = FastAPI(
     description="Pill verification against a patient's enrolled medications, pill identification and pictures from "
                 "a catalogue of US pills, and medication safety checks. A medication verification aid, not a medical "
                 "device: every alert defers to a pharmacist. Interaction data: DDInter 2.0 (CC BY-NC-SA 4.0); "
-                "ingredients: RxNorm (NLM); labels: openFDA; pill pictures: NLM C3PI (research and development use).",
+                "ingredients: RxNorm (NLM); labels: openFDA; pill pictures: NLM C3PI (research and development use). "
+                "Stage 2 answers are visual evidence only: they never confirm a medicine or that it is safe to take.",
+    responses={code: {"model": S.Error, "description": name} for code, name in ERROR_CODES.items()},
 )
 app.state.rxnorm = RxNormClient()
 app.state.safety_db = app.state.embedder = app.state.catalogue = app.state.feedback = None
@@ -178,10 +194,30 @@ async def api_key(request: Request, call_next):
     key = os.environ.get("EPILLS_API_KEY")
     if key and request.url.path.startswith("/v1") and request.method != "OPTIONS" \
             and request.headers.get("X-API-Key") != key:
-        return JSONResponse({"detail": "missing or wrong X-API-Key"}, status_code=401)
+        return JSONResponse({"code": "unauthorised", "detail": "missing or wrong X-API-Key"}, status_code=401)
     return await call_next(request)
 
 
+@app.exception_handler(StarletteHTTPException)
+async def http_error(_request, error):
+    """One error shape everywhere (blueprint B5): code, detail, and retry_after with Retry-After on 503."""
+    body, headers = {"code": ERROR_CODES.get(error.status_code, "error"), "detail": error.detail}, \
+        dict(getattr(error, "headers", None) or {})
+    if error.status_code == 503:
+        body["retry_after"], headers["Retry-After"] = RETRY_AFTER, str(RETRY_AFTER)
+    return JSONResponse(body, status_code=error.status_code, headers=headers)
+
+
+@app.exception_handler(RequestValidationError)
+async def invalid_input(_request, error):
+    return JSONResponse({"code": "invalid_input", "detail": jsonable(error.errors())}, status_code=422)
+
+
+def jsonable(errors):
+    return [{k: (v if isinstance(v, (str, int, float, list, type(None))) else str(v)) for k, v in e.items()}
+            for e in errors]
+
+
 # ---- request models ---------------------------------------------------------------------------------------------
 class Medication(BaseModel):
     id: str = Field(description="The app's id for this medication")
@@ -223,7 +259,9 @@ class GalleryMedication(BaseModel):
 class MatchRequest(BaseModel):
     queries: list[list[float]] = Field(min_length=1, description="Embeddings of the pills to check (/v1/embed)")
     gallery: list[GalleryMedication] = Field(min_length=1, description="The patient's enrolled medications")
-    model_version: str | None = Field(None, description="Version the gallery was embedded with; checked if given")
+    model_version: str | None = Field(None, description="The model_version stored with the gallery's embeddings. "
+                                                        "This or embedding_space_id is required")
+    embedding_space_id: str | None = Field(None, description="The embedding_space_id stored with the gallery")
     query_reads: list[list[str]] | None = Field(None, description="Per query, its `imprint_reads` from /v1/embed "
                                                                   "(read_imprint=true): turns on the imprint check")
 
@@ -287,6 +325,29 @@ def token_weights():
         return None
 
 
+def space(emb):
+    return getattr(emb, "embedding_space_id", emb.version)
+
+
+def check_gallery_version(emb, model_version, embedding_space_id):
+    """Embeddings compare only within one space (blueprint B3): the gallery must say which space it was made in."""
+    if not model_version and not embedding_space_id:
+        raise HTTPException(422, "send the model_version (or embedding_space_id) stored with the gallery's embeddings")
+    if (model_version and model_version != emb.version) or (embedding_space_id and embedding_space_id != space(emb)):
+        raise HTTPException(409, f"gallery embedded with {embedding_space_id or model_version}, but {space(emb)} is "
+                                 f"served: re-embed the enrolment photos with /v1/embed")
+
+
+def detection_out(det=None, boxes=None):
+    """The detection block: from the stand-in finder (det), or the boxes the app sent (trusted as given)."""
+    if det is None:
+        return {"boxes": [list(b) for b in boxes], "box_flags": [[] for _ in boxes], "pill_count": len(boxes),
+                "count_reliable": bool(boxes), "flags": [] if boxes else ["no_pills"], "found_by": "boxes sent"}
+    return {"boxes": [list(b) for b in det.boxes], "box_flags": det.box_flags, "pill_count": det.pill_count,
+            "count_reliable": bool(det.boxes) and "count_uncertain" not in det.flags, "flags": det.flags,
+            "found_by": STAND_IN}
+
+
 def match_decisions(queries, gallery, embedder, reads=None):
     """Per query: the decision, and with reads (per query, its OCR strings) the imprint check on it."""
     dim = queries.shape[1]
@@ -335,16 +396,22 @@ def product_summary(cat, pill_id, **extra):
 
 def identify_pill(embeddings, imprint=None, shape=None, colour=None, top_k=5, reads=None):
     cat = catalogue_for_identify()
-    candidates, notes = cat.identify(embeddings, top_k=top_k, imprint=imprint, shape=shape, colour=colour,
+    candidates, notes = cat.identify(embeddings, top_k=max(top_k, 2), imprint=imprint, shape=shape, colour=colour,
                                      reads=reads)
-    return {"candidates": [product_summary(cat, c.pop("pill_id"), **c) for c in candidates], "notes": notes,
-            "note": IDENTIFY_NOTE}
+    # calibrated on unfiltered searches only: with a typed imprint, shape or colour the confidence is left out
+    assessment = None if (imprint or shape or colour) else assess(candidates, cat.model_version)
+    return {"candidates": [product_summary(cat, c.pop("pill_id"), **c) for c in candidates[:top_k]], "notes": notes,
+            "note": IDENTIFY_NOTE, **(assessment or {"confidence": None, "catalogue_fit": None, "verdict": None,
+                                                     "verdict_message": None})}
 
 
 # ---- endpoints: health ----------------------------------------------------------------------------------------------
-@app.get("/health")
+@app.get("/health", response_model=S.Health)
 def health():
-    """Liveness check, and which assets are loaded (the model and catalogue load on first use)."""
+    """
+    Liveness check, and which parts are loaded. `ready` is true once the model, catalogue and safety database are
+    all loaded. A sleeping Space answers with an HTML page (not JSON) while it wakes up: retry until this is JSON.
+    """
     try:
         db = asset("safety_db")
         safety = {"loaded": True, "built_at": db.meta.get("built_at"), "pairs": db.meta.get("pairs")}
@@ -355,9 +422,12 @@ def health():
     device = getattr(getattr(emb, "forward", None), "device", None) or getattr(emb, "device", None)
     return {
         "status": "ok",
+        "ready": emb is not None and cat is not None and safety["loaded"],
         "api_version": API_VERSION,
+        "started_at": STARTED_AT,
         "model_repo": os.environ.get("EPILLS_MODEL_REPO", "not set"),
         "model": {"loaded": emb is not None, "version": getattr(emb, "version", None),
+                  "embedding_space_id": space(emb) if emb is not None else None,
                   "device": str(device) if device else None, "error": _errors.get("embedder")},
         "catalogue": {"loaded": cat is not None, "products": len(cat) if cat else None,
                       "model_version": getattr(cat, "model_version", None), "error": _errors.get("catalogue")},
@@ -367,8 +437,37 @@ def health():
     }
 
 
+@app.get("/v1/model-info", response_model=S.ModelInfo)
+def model_info():
+    """
+    The served model (blueprint B3): service version vs model version, the embedding space and preprocessing that
+    stored embeddings must match, the calibrated decision rule, and what the model was trained on and where it is weak.
+    """
+    emb = asset("embedder")
+    cal = emb.calibration
+    return {
+        "api_version": API_VERSION, "model_version": emb.version, "embedding_space_id": space(emb),
+        "preprocess_version": getattr(emb, "preprocess_version", None) or "unknown", "embedding_dim": 256,
+        "normalized": True, "crop_window": float(getattr(emb, "window", WINDOW)),
+        "threshold": cal["threshold"], "margin": cal["margin"],
+        "calibration": {"rule": cal.get("rule"), "calibrated_on": cal.get("calibrated_on")},
+        "imprint": {"gallery_min_weight": GALLERY_MIN_WEIGHT, "identify_alpha": IMPRINT_ALPHA,
+                    "identify_top_k": IMPRINT_TOP_K},
+        "training_coverage": "Loose-pill photos of US prescription pills (NLM C3PI): 2,362 medicines; catalogue of "
+                             "4,936 US pill products. Not trained on Singapore products, packaging or blisters.",
+        "known_limits": [
+            "Other phones and lighting: 48% top-1 over 32 Hong Kong products (MEDISEG), against 83% on held-out C3PI.",
+            "Similarities are not probabilities: pills that aren't in the catalogue still score about 0.93 median.",
+            "Imprints are readable on about 38% of photos.",
+            "The stand-in pill finder works on plain backgrounds only; check count_reliable before counting pills.",
+            "Visual evidence only: never confirms the medicine, its strength, or that it is safe to take.",
+        ],
+        "evidence": "visual",
+    }
+
+
 # ---- endpoints: Stage 2 -------------------------------------------------------------------------------------------
-@app.post("/v1/embed")
+@app.post("/v1/embed", response_model=S.Embedding)
 async def embed(image: UploadFile = File(description="A pill crop, or a photo with `box`"),
                 box: str | None = Form(None, description="Optional [x0, y0, x1, y1] of the pill in the photo; the "
                                                          "server then crops it with the crop contract"),
@@ -386,11 +485,13 @@ async def embed(image: UploadFile = File(description="A pill crop, or a photo wi
     vector = (await run_in_threadpool(emb, crops))[0]
     reads = await run_in_threadpool(imprint_reads, crops) if read_imprint else None
     return {"embedding": [round(float(v), 6) for v in vector], "dim": int(vector.shape[0]),
-            "model_version": emb.version, "framed_by": "box" if b else "as sent",
-            "imprint_reads": reads[0] if reads else None}
+            "model_version": emb.version, "embedding_space_id": space(emb),
+            "preprocess_version": getattr(emb, "preprocess_version", None) or "unknown", "normalized": True,
+            "framed_by": "box" if b else "as sent", "imprint_reads": reads[0] if reads else None,
+            "evidence": "visual"}
 
 
-@app.post("/v1/match")
+@app.post("/v1/match", response_model=S.MatchResponse)
 def match(request: MatchRequest):
     """
     **Which of my medicines is this?** Pill embeddings + the patient's gallery -> per pill: `match` (with the
@@ -398,18 +499,17 @@ def match(request: MatchRequest):
     The threshold and margin are the served model's calibration.
     """
     emb = asset("embedder")
-    if request.model_version and request.model_version != emb.version:
-        raise HTTPException(409, f"gallery embedded with model {request.model_version}, but {emb.version} is "
-                                 f"served: re-embed the enrolment photos with /v1/embed")
+    check_gallery_version(emb, request.model_version, request.embedding_space_id)
     queries = np.asarray(request.queries, np.float32)
     if request.query_reads is not None and len(request.query_reads) != len(queries):
         raise HTTPException(422, "query_reads needs one list of reads per query")
     return {"results": match_decisions(queries, request.gallery, emb, request.query_reads),
-            "model_version": emb.version,
-            "threshold": emb.calibration["threshold"], "margin": emb.calibration["margin"]}
+            "model_version": emb.version, "embedding_space_id": space(emb),
+            "threshold": emb.calibration["threshold"], "margin": emb.calibration["margin"],
+            "evidence": "visual", "limits": VISUAL_LIMITS}
 
 
-@app.post("/v1/identify")
+@app.post("/v1/identify", response_model=S.IdentifyResponse)
 async def identify(images: list[UploadFile] = File(description="One pill: 1 photo, or 2 (front and back)"),
                    boxes: str | None = Form(None, description="Optional [[x0, y0, x1, y1], ...], one per image"),
                    imprint: str | None = Form(None, description="Letters or numbers on the pill, e.g. 'M 547'"),
@@ -434,10 +534,24 @@ async def identify(images: list[UploadFile] = File(description="One pill: 1 phot
         per_crop = await run_in_threadpool(imprint_reads, crops)
         reads = [r for crop_reads in per_crop for r in crop_reads] if per_crop else None
     return {**identify_pill(vectors, imprint, shape, colour, top_k, reads), "imprint_reads": reads,
-            "model_version": emb.version}
+            "model_version": emb.version, "evidence": "visual", "limits": VISUAL_LIMITS}
 
 
-@app.post("/v1/scan")
+@app.post("/v1/detect", response_model=S.DetectResponse)
+async def detect(image: UploadFile = File(description="A full photo of the pills")):
+    """
+    **Where are the pills?** A photo -> a box per pill with quality flags (blueprint B6). Uses the stand-in finder for
+    plain backgrounds until the Stage 1 detector replaces it (same box format). Count pills only when
+    `count_reliable` is true; otherwise ask for a new photo (pills apart, plain surface).
+    """
+    photo = await read_image(image)
+    out = detection_out(await run_in_threadpool(detect_pills, photo))
+    if not out["boxes"]:
+        out["message"] = NO_PILL
+    return out
+
+
+@app.post("/v1/scan", response_model=S.ScanResponse)
 async def scan(image: UploadFile = File(description="A full photo of the pills"),
                boxes: str | None = Form(None, description="Pill boxes from a detector, [[x0, y0, x1, y1], ...]; "
                                                           "without them a stand-in finder looks for pills on a "
@@ -445,28 +559,34 @@ async def scan(image: UploadFile = File(description="A full photo of the pills")
                gallery: str | None = Form(None, description="JSON list of {medication_id, name, embeddings}: the "
                                                             "patient's gallery. Without it, each pill is "
                                                             "identified against the catalogue"),
-               schedule: str | None = Form(None, description="JSON list of {medication_id, count} for this dose: "
-                                                             "adds the dose check"),
-               model_version: str | None = Form(None, description="Version the gallery was embedded with"),
+               schedule: str | None = Form(None, description="Deprecated (the main backend owns dose rules): JSON "
+                                                             "list of {medication_id, count}; adds an advisory dose "
+                                                             "check"),
+               model_version: str | None = Form(None, description="The model_version stored with the gallery "
+                                                                  "(this or embedding_space_id is required with one)"),
+               embedding_space_id: str | None = Form(None, description="The embedding_space_id stored with the "
+                                                                       "gallery"),
                top_k: int = Form(3, ge=1, le=10),
                read_imprint: bool = Form(True, description="Read each pill's imprint: checks the gallery decisions "
                                                            "(medications with an `imprint`), or re-ranks catalogue "
                                                            "candidates")):
     """
-    **One call for a dose check:** full photo -> every pill's box and decision (with the patient's gallery), or its
-    catalogue candidates (without one); with a schedule, also the dose-check alerts of `/v1/safety/dose`.
+    **Candidates for every pill in a photo** (diagnostics; blueprint D3): full photo -> each pill's box with quality
+    flags and its visual decision (with the patient's gallery) or catalogue candidates (without one). Not a dose
+    verification: the main backend decides with the regimen. Boxes flagged by the finder are not counted.
     """
     emb = asset("embedder")
     photo = await read_image(image)
     box_list = parse_boxes(boxes)
-    found_by = "boxes sent"
     if box_list is None:
-        from epills.core.inference.pill_finder import find_pills
-        box_list, found_by = find_pills(photo), "stand-in finder (plain backgrounds only)"
-    result = {"found_by": found_by, "model_version": emb.version, "pills": []}
+        detection = detection_out(await run_in_threadpool(detect_pills, photo))
+    else:
+        detection = detection_out(boxes=box_list)
+    box_list = [tuple(b) for b in detection["boxes"]]
+    result = {"found_by": detection["found_by"], "model_version": emb.version, "detection": detection, "pills": [],
+              "evidence": "visual", "limits": VISUAL_LIMITS}
     if not box_list:
-        result["message"] = ("No pill found. Spread the pills apart on a plain background and photograph them "
-                             "again.")
+        result["message"] = NO_PILL
         return result
     crops = emb.frame(photo, box_list)
     vectors = await run_in_threadpool(emb, crops)  # in a thread: a GPU queue wait mustn't block the API
@@ -476,33 +596,49 @@ async def scan(image: UploadFile = File(description="A full photo of the pills")
             meds = [GalleryMedication(**g) for g in parse_json(gallery, "gallery")]
         except (TypeError, ValidationError) as error:
             raise HTTPException(422, f"gallery: {error}")
-        if model_version and model_version != emb.version:
-            raise HTTPException(409, f"gallery embedded with model {model_version}, but {emb.version} is served")
+        check_gallery_version(emb, model_version, embedding_space_id)
     reads = None
     if read_imprint and (meds is None or any(m.imprint for m in meds)):
         reads = await run_in_threadpool(imprint_reads, crops)
 
     if meds is not None:
         decisions = match_decisions(vectors, meds, emb, reads)
-        result["pills"] = [{"box": list(b), **d, "imprint_reads": reads[i] if reads else None}
-                           for i, (b, d) in enumerate(zip(box_list, decisions))]
+        result["pills"] = [{"box": list(b), "quality_flags": fl, "counted": not fl, **d,
+                            "imprint_reads": reads[i] if reads else None}
+                           for i, (b, fl, d) in enumerate(zip(box_list, detection["box_flags"], decisions))]
         if schedule:
             try:
                 sched = [ScheduledMedication(**s).model_dump() for s in parse_json(schedule, "schedule")]
             except (TypeError, ValidationError) as error:
                 raise HTTPException(422, f"schedule: {error}")
-            alerts, summary = check_dose(sched, [{"medication_id": d["medication_id"], "decision": d["decision"]}
-                                                 for d in decisions],
-                                         {m.medication_id: m.name or m.medication_id for m in meds})
-            result["dose"] = {"alerts": [a.to_dict() for a in alerts], "summary": summary}
+            result["dose"] = advisory_dose(sched, result["pills"], detection, meds)
     else:
-        result["pills"] = [{"box": list(b), **identify_pill(v[None], top_k=top_k, reads=reads[i] if reads else None),
+        result["pills"] = [{"box": list(b), "quality_flags": fl, "counted": not fl,
+                            **identify_pill(v[None], top_k=top_k, reads=reads[i] if reads else None),
                             "imprint_reads": reads[i] if reads else None}
-                           for i, (b, v) in enumerate(zip(box_list, vectors))]
+                           for i, (b, fl, v) in enumerate(zip(box_list, detection["box_flags"], vectors))]
     return result
 
 
-@app.post("/v1/read-label")
+def advisory_dose(sched, pills, detection, meds):
+    """/v1/scan's deprecated dose check: only when the pill count is reliable, and only counted pills."""
+    note = ("Advisory and deprecated: the main backend checks doses against the regimen and dose history "
+            "(docs/dose_rules.md).")
+    if not detection["count_reliable"]:
+        alert = Alert("count_uncertain", "moderate", "I can't count the pills in this photo reliably. Spread them "
+                      "apart on a plain surface and photograph them again.", [], SOURCE_SCHEDULE,
+                      details={"flags": detection["flags"]})
+        summary = {"pills_seen": len(pills), "matched": 0, "unknown": 0, "unsure": 0, "all_ok": False,
+                   "all_visually_matched": False, "count_reliable": False,
+                   "text": "I can't count the pills in this photo reliably."}
+        return {"alerts": [alert.to_dict()], "summary": summary, "deprecated": True, "note": note}
+    alerts, summary = check_dose(sched, [{"medication_id": p["medication_id"], "decision": p["decision"]}
+                                         for p in pills if p["counted"]],
+                                 {m.medication_id: m.name or m.medication_id for m in meds})
+    return {"alerts": [a.to_dict() for a in alerts], "summary": summary, "deprecated": True, "note": note}
+
+
+@app.post("/v1/read-label", response_model=S.ReadLabelResponse)
 async def read_label(image: UploadFile = File(description="A photo of a medicine box or a pharmacy label, close "
                                                           "up and straight on")):
     """
@@ -525,7 +661,7 @@ async def read_label(image: UploadFile = File(description="A photo of a medicine
     return result
 
 
-@app.get("/v1/pills/search")
+@app.get("/v1/pills/search", response_model=S.SearchResponse)
 def pills_search(name: str = Query(min_length=2, description="Brand or generic name, e.g. 'metformin', 'Glucophage'"),
                  strength: str | None = Query(None, description="e.g. '500 MG'"),
                  limit: int = Query(50, ge=1, le=200)):
@@ -541,7 +677,7 @@ def pills_search(name: str = Query(min_length=2, description="Brand or generic n
             "source": cat.meta.get("source")}
 
 
-@app.get("/v1/pills/{pill_id}")
+@app.get("/v1/pills/{pill_id}", response_model=S.Product)
 def pill(pill_id: str):
     """One catalogue product: names, ingredients, appearance, imprint and its pictures."""
     cat = asset("catalogue")
@@ -560,7 +696,7 @@ def pill_image(pill_id: str, n: int):
 
 
 # ---- endpoints: feedback for retraining (opt-in) -----------------------------------------------------------------
-@app.post("/v1/feedback")
+@app.post("/v1/feedback", response_model=S.FeedbackStored)
 async def feedback(image: UploadFile = File(description="The pill: a crop, or a photo with `box`"),
                    box: str | None = Form(None, description="Optional [x0, y0, x1, y1] of the pill in the photo"),
                    consent: bool = Form(description="The user agreed to share this photo to improve the model"),
@@ -595,7 +731,7 @@ async def feedback(image: UploadFile = File(description="The pill: a crop, or a
     return {"stored": True, "item": item, "note": "Thank you. The photo is used only to improve pill recognition."}
 
 
-@app.delete("/v1/feedback/{consent_id}")
+@app.delete("/v1/feedback/{consent_id}", response_model=S.FeedbackDeleted)
 def feedback_delete(consent_id: str):
     """Withdrawal of consent: deletes every photo stored under this consent id."""
     try:
@@ -605,7 +741,7 @@ def feedback_delete(consent_id: str):
 
 
 # ---- endpoints: Stage 3 -------------------------------------------------------------------------------------------
-@app.post("/v1/safety/medications")
+@app.post("/v1/safety/medications", response_model=S.MedicationsResponse)
 def safety_medications(request: MedicationsRequest):
     """
     Run when a medication is added or changed: interactions between every pair of the patient's medications
@@ -624,18 +760,19 @@ def safety_medications(request: MedicationsRequest):
     }
 
 
-@app.post("/v1/safety/dose")
+@app.post("/v1/safety/dose", response_model=S.DoseResponse)
 def safety_dose(request: DoseRequest):
     """
-    Run at every dose check, after /v1/match: double doses, pills not part of this dose, missed medications, and
-    unknown or unsure pills. (/v1/scan with a schedule does this in the same call.)
+    The dose rules' **reference implementation** (blueprint D3: the main backend owns the decision, with the regimen
+    and dose history; spec and test cases in docs/dose_rules.md). After /v1/match: extra pills, pills not part of
+    this dose, scheduled medicines not seen, and unknown or unsure pills. Visual evidence only.
     """
     alerts, summary = check_dose([s.model_dump() for s in request.schedule],
                                  [p.model_dump() for p in request.pills], request.names)
     return {"alerts": [a.to_dict() for a in alerts], "summary": summary}
 
 
-@app.get("/v1/safety/label")
+@app.get("/v1/safety/label", response_model=S.LabelResponse)
 def safety_label(rxcui: str | None = Query(None), ingredient: str | None = Query(None)):
     """The warning sections of the official US label (openFDA), quoted as written; only when the user asks."""
     if not rxcui and not ingredient:

serving/app/schemas.py

@@ -0,0 +1,287 @@
+"""
+The shape of every PharmaScan API answer (blueprint job B5), so /openapi.json documents responses, not only requests.
+Models allow extra fields: new fields can be added without breaking clients that validate against these.
+"""
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+Evidence = Literal["visual"]
+
+
+class Out(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+
+class Error(Out):
+    code: str = Field(description="unauthorised, not_found, version_mismatch, invalid_input, upstream_failed, "
+                                  "not_ready or error")
+    detail: str | list | dict = Field(description="What went wrong (for invalid_input: the fields at fault)")
+    retry_after: int | None = Field(None, description="Seconds to wait before retrying (with not_ready)")
+
+
+# ---- health and model info -------------------------------------------------------------------------------------------
+class Part(Out):
+    loaded: bool
+    error: str | None = None
+
+
+class ModelPart(Part):
+    version: str | None = None
+    embedding_space_id: str | None = None
+    device: str | None = Field(None, description="zerogpu, or cpu (slower, when no GPU is granted)")
+
+
+class CataloguePart(Part):
+    products: int | None = None
+    model_version: str | None = None
+
+
+class SafetyPart(Part):
+    built_at: str | None = None
+    pairs: str | None = None
+
+
+class TextPart(Part):
+    model: str | None = None
+
+
+class Health(Out):
+    status: Literal["ok"]
+    ready: bool = Field(description="The model, catalogue and safety database are all loaded")
+    api_version: str
+    started_at: str
+    model_repo: str
+    model: ModelPart
+    catalogue: CataloguePart
+    safety_db: SafetyPart
+    text_reading: TextPart
+
+
+class ModelInfo(Out):
+    api_version: str
+    model_version: str
+    embedding_space_id: str = Field(description="Compare embeddings only within one space")
+    preprocess_version: str
+    embedding_dim: int
+    normalized: bool
+    crop_window: float = Field(description="Crop contract: square window = this x the pill box's long side")
+    threshold: float
+    margin: float
+    calibration: dict
+    imprint: dict
+    training_coverage: str
+    known_limits: list[str]
+    evidence: Evidence
+
+
+# ---- Stage 2 ---------------------------------------------------------------------------------------------------------
+class Embedding(Out):
+    embedding: list[float]
+    dim: int
+    model_version: str
+    embedding_space_id: str
+    preprocess_version: str
+    normalized: bool
+    framed_by: Literal["box", "as sent"]
+    imprint_reads: list[str] | None = None
+    evidence: Evidence
+
+
+class Ref(Out):
+    medication_id: str
+    name: str
+    similarity: float
+
+
+class MatchResult(Out):
+    decision: Literal["match", "unknown", "ambiguous"]
+    medication_id: str | None
+    name: str | None
+    similarity: float
+    closest: Ref
+    runner_up: Ref | None
+    message: str
+    imprint_check: str | None = None
+    imprint_read_for: list[str] = []
+
+
+class MatchResponse(Out):
+    results: list[MatchResult]
+    model_version: str
+    embedding_space_id: str
+    threshold: float
+    margin: float
+    evidence: Evidence
+    limits: str
+
+
+class Picture(Out):
+    url: str
+    kind: str
+
+
+class Product(Out):
+    pill_id: str
+    name: str | None
+    rxnorm_name: str | None
+    generic_name: str | None
+    manufacturer: str | None
+    ingredients: list[str]
+    shape: str | None
+    colour: str | None
+    size_mm: float | None
+    imprint: str | None
+    images: list[Picture]
+
+
+class Candidate(Product):
+    similarity: float = Field(description="Cosine similarity of appearance: not a probability")
+    imprint_match: bool = Field(description="The typed imprint fits this product")
+    imprint_read: float = Field(description="How well the imprint read on the photo fits this product, 0-1")
+
+
+class Identification(Out):
+    candidates: list[Candidate]
+    notes: list[str]
+    note: str
+    confidence: float | None = Field(None, description="Chance the first candidate is the right product, if the pill "
+                                                       "is a US catalogue product (calibrated; null with filters)")
+    catalogue_fit: Literal["good", "weak", "poor"] | None = None
+    verdict: Literal["possible_match", "uncertain", "no_confident_match"] | None = None
+    verdict_message: str | None = None
+
+
+class IdentifyResponse(Identification):
+    imprint_reads: list[str] | None = None
+    model_version: str
+    evidence: Evidence
+    limits: str
+
+
+class Detection(Out):
+    boxes: list[list[float]] = Field(description="[x0, y0, x1, y1] per pill, in the photo's pixels")
+    box_flags: list[list[str]] = Field(description="Per box: touches_edge, merged, irregular, small")
+    pill_count: int
+    count_reliable: bool = Field(description="False when any box is flagged or blobs were left out: don't count "
+                                             "pills from this photo")
+    flags: list[str] = Field(description="Per photo: no_pills, count_uncertain, too_many")
+    found_by: str
+
+
+class DetectResponse(Detection):
+    message: str | None = None
+
+
+class Alert(Out):
+    type: str
+    severity: str
+    message: str
+    medication_ids: list[str]
+    source: str
+    severity_graded: bool
+    colour: str
+    speak: bool
+    notify_caregiver: bool
+    details: dict
+
+
+class DoseSummary(Out):
+    pills_seen: int
+    matched: int
+    unknown: int
+    unsure: int
+    all_ok: bool = Field(description="Deprecated name of all_visually_matched")
+    all_visually_matched: bool
+    count_reliable: bool = True
+    text: str
+
+
+class DoseResponse(Out):
+    alerts: list[Alert]
+    summary: DoseSummary
+
+
+class ScanDose(DoseResponse):
+    deprecated: bool = True
+    note: str
+
+
+class ScanPill(Out):
+    box: list[float]
+    quality_flags: list[str] = []
+    counted: bool = Field(True, description="False for a flagged box: not counted in the dose check")
+    imprint_reads: list[str] | None = None
+
+
+class ScanResponse(Out):
+    found_by: str
+    model_version: str
+    detection: Detection
+    pills: list[ScanPill] = Field(description="With a gallery: /v1/match's fields per pill; without: /v1/identify's")
+    dose: ScanDose | None = None
+    message: str | None = None
+    evidence: Evidence
+    limits: str
+
+
+class Ingredient(Out):
+    rxcui: str | None
+    name: str
+
+
+class LabelCandidate(Out):
+    name: str
+    brand: str | None
+    ingredients: list[Ingredient]
+    strength: str | None
+    rxcui: str | None
+    rxnorm_name: str | None
+    found_in: list[str] = Field(description="Only the lines that name the medicine")
+
+
+class ReadLabelResponse(Out):
+    candidates: list[LabelCandidate]
+    note: str
+    ocr_model: str
+    message: str | None = None
+
+
+class SearchProduct(Product):
+    match: Literal["same", "contains", "name"]
+
+
+class SearchResponse(Out):
+    resolution: dict
+    count: int
+    products: list[SearchProduct]
+    source: str | None
+
+
+# ---- Stage 3 and feedback --------------------------------------------------------------------------------------------
+class MedicationsResponse(Out):
+    alerts: list[Alert]
+    medications: list[dict]
+    all_identified: bool
+    sources: dict
+
+
+class LabelResponse(Out):
+    brand_name: str | None
+    generic_name: str | None
+    label_date: str | None
+    matched_by: str
+    sections: dict[str, str]
+    disclaimer: str
+
+
+class FeedbackStored(Out):
+    stored: bool
+    item: str
+    note: str
+
+
+class FeedbackDeleted(Out):
+    deleted: bool
+    local_photos_removed: int
+    removed_from_repo: bool

src/epills/core/catalogue/confidence.py

@@ -0,0 +1,43 @@
+"""
+Honest confidence for catalogue identification (blueprint job B4). Raw similarities are not probabilities: pills that
+aren't in the catalogue at all still score about 0.93. Fitted by scripts/calibrate_identify.py for one model; for any
+other model version, assess() returns None (refit after every new model).
+
+  confidence     chance the first candidate is the right product *if the pill is a US catalogue product*: logistic
+                 fit of (top-1 similarity, gap to the runner-up, imprint read on the top-1) on C3PI validation photos
+  catalogue_fit  poor / weak / good: poor below what 95% of rightly identified catalogue pills reach, good above what
+                 90% of non-catalogue pills (MEDISEG) reach
+  verdict        no_confident_match (poor fit or confidence < LOW), possible_match (good fit and confidence >= HIGH),
+                 uncertain (the rest)
+Test (run 4, held-out medicines; outputs/runs/stage2_run4/eval/identify_confidence.json): stated 0.93 -> 95% right,
+0.71 -> 66%, 0.53 -> 48%, 0.31 -> 33%, 0.03 -> 3%. Catalogue pills: possible_match 73% (83% right), uncertain 11%,
+no_confident_match 16%. Non-catalogue pills: no_confident_match 71%, uncertain 21%, possible_match 9%.
+"""
+import math
+
+MODEL_VERSION = "stage2_run4@2026-10-09"
+WEIGHTS = (-36.3393, 37.514, 31.6165, 1.0213)  # intercept, top-1 similarity, gap to the runner-up, imprint read
+T_POOR, T_GOOD = 0.9451, 0.9654
+LOW, HIGH = 0.2, 0.5
+MESSAGES = {
+    "possible_match": "The first candidate looks most like this pill. A photo can't confirm it. Check with your "
+                      "pharmacist.",
+    "uncertain": "Several catalogue pills look alike and none stands out. Type the imprint, or check with your "
+                 "pharmacist.",
+    "no_confident_match": "No catalogue pill matches this one well: it may not be a US catalogue product, or the photo "
+                          "differs from the catalogue photos. Check with your pharmacist.",
+}
+
+
+def assess(candidates, model_version):
+    """candidates: best first, each with similarity and imprint_read (at least two) -> dict, or None."""
+    if model_version != MODEL_VERSION or len(candidates) < 2:
+        return None
+    s1, s2 = candidates[0]["similarity"], candidates[1]["similarity"]
+    z = WEIGHTS[0] + WEIGHTS[1] * s1 + WEIGHTS[2] * (s1 - s2) + WEIGHTS[3] * candidates[0].get("imprint_read", 0.0)
+    confidence = 1 / (1 + math.exp(-z))
+    fit = "poor" if s1 < T_POOR else "good" if s1 >= T_GOOD else "weak"
+    verdict = ("no_confident_match" if fit == "poor" or confidence < LOW else
+               "possible_match" if fit == "good" and confidence >= HIGH else "uncertain")
+    return {"confidence": round(confidence, 3), "catalogue_fit": fit, "verdict": verdict,
+            "verdict_message": MESSAGES[verdict]}

src/epills/core/inference/embedder.py

@@ -9,7 +9,7 @@ import os
 import numpy as np
 import torch
 
-from epills.core.inference.preprocess import WINDOW, crop_window, eval_transform
+from epills.core.inference.preprocess import CROP_SIZE, IMAGE_SIZE, WINDOW, crop_window, eval_transform
 from epills.core.models.hybrid import HybridPillNet
 
 
@@ -22,6 +22,9 @@ class Embedder:
         self.version = ckpt.get("version") or ckpt.get("config", {}).get("name", "unversioned")
         self.calibration = ckpt.get("calibration") or {}
         self.window = self.calibration.get("window") or WINDOW  # the crop contract this model was calibrated with
+        # embeddings are comparable only within one space: same model, same crop contract, loose-pill photos
+        self.embedding_space_id = f"pill-crop:{self.version}:w{self.window:g}"
+        self.preprocess_version = f"crop-contract-1:window-{self.window:g}:crop-{CROP_SIZE}:input-{IMAGE_SIZE}"
         self.transform = eval_transform()
         self.batch_size = batch_size
 

src/epills/core/inference/pill_finder.py

@@ -1,46 +1,190 @@
 """
 A stand-in for the Stage 1 detector: finds pills on a plain background (a mat, a table) in a full photo.
 
-Works when the background is one roughly uniform colour and pills don't touch. Anything else needs the real
-detector, whose boxes the API accepts instead. The background is the median colour of a frame along the photo's
-edges; a pill is a blob that differs from it in Lab colour, after a morphological clean-up and hole filling.
+Works when the background is plain (one colour, possibly unevenly lit) and pills lie apart. Anything else needs the
+real detector, whose boxes the API accepts instead.
+1. Background: each pixel is compared with the mat's colour *at that place*: a smooth surface (degree-3 polynomial
+   per Lab channel) fitted robustly, so vignetting and lighting gradients don't turn corners into "pills". Pills are
+   outliers to that fit and don't move it.
+2. A pill is a blob that differs from it (Otsu threshold, at least MIN_DISTANCE), after a morphological clean-up and
+   hole filling, with a pill-like shape: compact (solidity), not too elongated, not a sliver along the photo's edge.
+3. Pieces of one pill (a light pill on a light background splits along its score line or imprint) are merged when
+   they nearly touch; small crumbs next to a pill are dropped.
+Every box carries quality flags; any flag, or a dropped crumb, sets the photo flag `count_uncertain`, so a pill count
+from this finder is never trusted blindly (blueprint job B1).
 """
+from dataclasses import dataclass, field
+
 import cv2
 import numpy as np
 
 WORK = 768             # long side of the working image
-MIN_DISTANCE = 14.0    # Lab distance below which a pixel is background
-MIN_AREA = 0.0015      # smallest blob, as a share of the photo
-MAX_AREA = 0.35        # largest blob (bigger: probably the mat's edge or a shadow, not a pill)
+MIN_DISTANCE = 12.0    # Lab distance from the local background below which a pixel is background
+MIN_AREA = 0.0008      # smallest blob, as a share of the photo
+MAX_AREA = 0.25        # largest blob
+MIN_SOLIDITY = 0.80    # blob area / its convex hull: pills are convex, lighting patches and shadows aren't
+MAX_ASPECT = 4.5       # long / short side of the blob's rotated rectangle
+MERGE_GAP = 0.12       # pieces closer than this x the larger piece's long side are one pill
+CRUMB_RATIO = 0.2      # a blob smaller than this x a neighbouring pill's area ...
+CRUMB_GAP = 0.6        # ... within this x its long side is a crumb (shadow edge, chip), dropped
+TEXTURE_RATIO = 0.08   # anywhere, a blob below this x the largest pill's area is a speck of texture, dropped
+LOW_FACTOR = 0.5       # hysteresis: the faint threshold, as a share of the clear one
 
 
-def find_pills(image, max_pills=20):
-    """PIL photo -> boxes (x0, y0, x1, y1) in the photo's pixels, largest first."""
-    rgb = np.asarray(image.convert("RGB"))
-    h, w = rgb.shape[:2]
-    f = WORK / max(h, w)
-    small = cv2.resize(rgb, (max(1, round(w * f)), max(1, round(h * f))), interpolation=cv2.INTER_AREA)
-    lab = cv2.cvtColor(small, cv2.COLOR_RGB2LAB).astype(np.float32)
+@dataclass
+class Detection:
+    boxes: list = field(default_factory=list)       # (x0, y0, x1, y1) in the photo's pixels, largest first
+    box_flags: list = field(default_factory=list)   # per box: "touches_edge", "merged", "irregular", "small"
+    flags: list = field(default_factory=list)       # per photo: "no_pills", "count_uncertain", "too_many"
+    dropped: int = 0                                # crumbs and non-pill blobs left out
+
+    @property
+    def pill_count(self):
+        return len(self.boxes)
+
+
+def _gap(a, b):
+    """Distance between two boxes (0 when they overlap)."""
+    dx = max(a[0] - b[2], b[0] - a[2], 0.0)
+    dy = max(a[1] - b[3], b[1] - a[3], 0.0)
+    return float(np.hypot(dx, dy))
+
+
+def _long(b):
+    return max(b[2] - b[0], b[3] - b[1])
+
+
+def _area(b):
+    return (b[2] - b[0]) * (b[3] - b[1])
+
+
+def _terms(x, y):
+    """Polynomial terms up to degree 3 in x and y (in -1..1): enough for vignetting and lighting gradients."""
+    return np.stack([np.ones_like(x), x, y, x * x, x * y, y * y, x ** 3, x * x * y, x * y * y, y ** 3], axis=-1)
+
+
+def _background(lab):
+    """The mat's colour at every pixel: a smooth surface per Lab channel, fitted robustly (pills are outliers)."""
+    sh, sw = lab.shape[:2]
+    tiny = cv2.resize(lab, (64, max(8, round(64 * sh / sw))), interpolation=cv2.INTER_AREA).astype(np.float32)
+    th, tw = tiny.shape[:2]
+    gy, gx = np.mgrid[0:th, 0:tw]
+    A = _terms(gx / (tw - 1) * 2 - 1, gy / (th - 1) * 2 - 1).reshape(-1, 10)
+    Y = tiny.reshape(-1, 3)
+    keep = np.ones(len(Y), bool)
+    for _ in range(4):  # refit without the pixels that don't look like mat
+        coef, *_ = np.linalg.lstsq(A[keep], Y[keep], rcond=None)
+        resid = np.linalg.norm(Y - A @ coef, axis=1)
+        keep = resid <= max(2.5 * np.median(resid[keep]), 3.0)
+    gy, gx = np.mgrid[0:sh, 0:sw]
+    return (_terms(gx / (sw - 1) * 2 - 1, gy / (sh - 1) * 2 - 1) @ coef).astype(np.float32)
+
+
+def _mask(lab):
+    """Pixels that differ from the mat's colour at that place."""
     sh, sw = lab.shape[:2]
-    b = max(2, int(0.04 * min(sh, sw)))
-    frame = np.concatenate([lab[:b].reshape(-1, 3), lab[-b:].reshape(-1, 3),
-                            lab[:, :b].reshape(-1, 3), lab[:, -b:].reshape(-1, 3)])
-    d = np.linalg.norm(lab - np.median(frame, axis=0), axis=2)
+    d = np.linalg.norm(lab.astype(np.float32) - _background(lab), axis=2)
     otsu, _ = cv2.threshold(np.clip(d * 2, 0, 255).astype(np.uint8), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
-    mask = (d > max(otsu / 2, MIN_DISTANCE)).astype(np.uint8)
+    high = max(otsu / 2, MIN_DISTANCE)
     k = max(3, int(0.01 * min(sh, sw)) | 1)
     kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))
-    mask = cv2.morphologyEx(cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel), cv2.MORPH_CLOSE, kernel)
+    strong = cv2.morphologyEx((d > high).astype(np.uint8), cv2.MORPH_OPEN, kernel)
+    # hysteresis: a pill's faint parts (the white half of a capsule) join the clearly different part they touch
+    weak = cv2.morphologyEx((d > max(high * LOW_FACTOR, MIN_DISTANCE * 0.75)).astype(np.uint8), cv2.MORPH_OPEN, kernel)
+    n, labels = cv2.connectedComponents(weak, connectivity=8)
+    keep = np.zeros(n, bool)
+    keep[np.unique(labels[strong > 0])] = True
+    keep[0] = False
+    mask = cv2.morphologyEx(keep[labels].astype(np.uint8), cv2.MORPH_CLOSE, kernel)
     flood = np.pad(mask, 1)
     cv2.floodFill(flood, np.zeros((sh + 4, sw + 4), np.uint8), (0, 0), 1)
-    mask = mask | (1 - flood[1:-1, 1:-1])
+    return mask | (1 - flood[1:-1, 1:-1])
+
+
+def detect_pills(image, max_pills=20):
+    """PIL photo -> Detection: pill boxes in the photo's pixels with quality flags."""
+    rgb = np.asarray(image.convert("RGB"))
+    h, w = rgb.shape[:2]
+    f = WORK / max(h, w)
+    small = cv2.resize(rgb, (max(1, round(w * f)), max(1, round(h * f))), interpolation=cv2.INTER_AREA)
+    mask = _mask(cv2.cvtColor(small, cv2.COLOR_RGB2LAB))
+    sh, sw = mask.shape
 
-    n, _, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8)
-    boxes = []
-    for x, y, bw, bh, area in stats[1:]:
+    blobs, dropped = [], 0
+    n, labels, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8)
+    for i in range(1, n):
+        x, y, bw, bh, area = stats[i]
         share = area / (sh * sw)
-        touches = (x <= 1) + (y <= 1) + (x + bw >= sw - 1) + (y + bh >= sh - 1)
-        if MIN_AREA <= share <= MAX_AREA and touches < 2:
-            boxes.append((area, (x / f, y / f, (x + bw) / f, (y + bh) / f)))
-    boxes.sort(key=lambda t: -t[0])
-    return [tuple(round(v, 1) for v in box) for _, box in boxes[:max_pills]]
+        if not MIN_AREA <= share <= MAX_AREA:
+            dropped += share > MAX_AREA
+            continue
+        contours, _ = cv2.findContours((labels[y:y + bh, x:x + bw] == i).astype(np.uint8), cv2.RETR_EXTERNAL,
+                                       cv2.CHAIN_APPROX_SIMPLE)
+        contour = max(contours, key=cv2.contourArea)
+        hull = cv2.contourArea(cv2.convexHull(contour))
+        solidity = area / hull if hull > 0 else 0.0
+        (_, _), (rw, rh), _ = cv2.minAreaRect(contour)
+        aspect = max(rw, rh) / max(min(rw, rh), 1.0)
+        edges = (x <= 1) + (y <= 1) + (x + bw >= sw - 1) + (y + bh >= sh - 1)
+        if edges >= 2 or aspect > MAX_ASPECT or (edges and solidity < 0.9) or solidity < MIN_SOLIDITY - 0.15:
+            dropped += 1  # lighting patches, mat edges, slivers: not pills
+            continue
+        flags = set()
+        if edges:
+            flags.add("touches_edge")
+        if solidity < MIN_SOLIDITY:
+            flags.add("irregular")
+        blobs.append({"box": [float(x), float(y), float(x + bw), float(y + bh)], "area": float(area), "flags": flags})
+
+    merged = True
+    while merged:  # pieces of one pill
+        merged = False
+        for a in range(len(blobs)):
+            for b in range(a + 1, len(blobs)):
+                A, B = blobs[a], blobs[b]
+                if _gap(A["box"], B["box"]) <= MERGE_GAP * max(_long(A["box"]), _long(B["box"])):
+                    A["box"] = [min(A["box"][0], B["box"][0]), min(A["box"][1], B["box"][1]),
+                                max(A["box"][2], B["box"][2]), max(A["box"][3], B["box"][3])]
+                    A["area"] += B["area"]
+                    A["flags"] |= B["flags"] | {"merged"}
+                    del blobs[b]
+                    merged = True
+                    break
+            if merged:
+                break
+
+    if any("touches_edge" not in b["flags"] for b in blobs):  # whole pills found: blobs cut by the edge are mat
+        dropped += sum("touches_edge" in b["flags"] for b in blobs)
+        blobs = [b for b in blobs if "touches_edge" not in b["flags"]]
+    blobs.sort(key=lambda t: -t["area"])
+    kept = []
+    for blob in blobs:  # crumbs next to a bigger pill, specks of texture anywhere
+        if kept and (blob["area"] < TEXTURE_RATIO * kept[0]["area"] or any(
+                blob["area"] < CRUMB_RATIO * k["area"] and _gap(blob["box"], k["box"]) < CRUMB_GAP * _long(k["box"])
+                for k in kept)):
+            dropped += 1
+            continue
+        kept.append(blob)
+    if len(kept) >= 2:
+        median = float(np.median([k["area"] for k in kept]))
+        for k in kept:
+            if k["area"] < 0.15 * median:
+                k["flags"].add("small")
+
+    det = Detection(dropped=dropped)
+    for k in kept[:max_pills]:
+        det.boxes.append(tuple(round(v / f, 1) for v in k["box"]))
+        det.box_flags.append(sorted(k["flags"]))
+    if not det.boxes:
+        det.flags.append("no_pills")
+    if len(kept) > max_pills:
+        det.flags.append("too_many")
+    if any(det.box_flags) or len(kept) > max_pills or any(
+            "merged" in fl for fl in det.box_flags):
+        det.flags.append("count_uncertain")
+    return det
+
+
+def find_pills(image, max_pills=20):
+    """PIL photo -> boxes (x0, y0, x1, y1) in the photo's pixels, largest first (see detect_pills for flags)."""
+    return detect_pills(image, max_pills).boxes

src/epills/core/safety/rules.py

@@ -1,251 +1,254 @@
-"""
-Stage 3 rules: fixed logic on curated data, no trained model (ROADMAP §4).
-
-  check_medications  when a medication is added or changed: drug-drug interactions (DDInter 2.0) and therapeutic
-                     duplication (two medications sharing an active ingredient, via RxNorm)
-  check_dose         at every dose check: double doses, pills not part of this dose, missed medications, unknown and
-                     unsure pills (from Stage 2's matching)
-
-Alert rules from the project plan:
-  - Major: red, spoken, caregiver notified. Moderate: amber, spoken. Minor: shown on screen only.
-  - Every alert ends with "Check with your pharmacist." The app never tells anyone to take or stop a medication.
-  - DDInter's "Unknown" level (listed, but not graded) gets its own severity, "ungraded": on screen only, not spoken
-    (decision D6, 2026-10-08), so common medication lists don't drown the graded alerts in noise.
-  - A medication that can't be identified is reported, never silently skipped: no alert must not mean "no
-    interaction" when the check couldn't run.
-"""
-from dataclasses import asdict, dataclass, field
-from itertools import combinations
-
-from epills.core.safety.db import normalise_name
-
-PHARMACIST = "Check with your pharmacist."
-UNKNOWN_LEVEL_AS = "ungraded"  # decision D6
-SOURCE_DDINTER = "DDInter 2.0 (CC BY-NC-SA 4.0), interaction severity"
-SOURCE_RXNORM = "RxNorm (NLM), active ingredients"
-SOURCE_SCHEDULE = "The patient's schedule and the pill check"
-
-STYLE = {  # severity -> how the app shows it (the plan's severity mapping)
-    "major": {"colour": "red", "speak": True, "notify_caregiver": True},
-    "moderate": {"colour": "amber", "speak": True, "notify_caregiver": False},
-    "ungraded": {"colour": "grey", "speak": False, "notify_caregiver": False},
-    "minor": {"colour": "grey", "speak": False, "notify_caregiver": False},
-}
-RANK = {"major": 4, "moderate": 3, "ungraded": 2, "minor": 1}
-
-
-@dataclass
-class Alert:
-    type: str
-    severity: str
-    message: str
-    medication_ids: list
-    source: str
-    severity_graded: bool = True
-    colour: str = ""
-    speak: bool = False
-    notify_caregiver: bool = False
-    details: dict = field(default_factory=dict)
-
-    def __post_init__(self):
-        style = STYLE[self.severity]
-        self.colour, self.speak, self.notify_caregiver = style["colour"], style["speak"], style["notify_caregiver"]
-        if not self.message.endswith(PHARMACIST):
-            self.message = f"{self.message} {PHARMACIST}"
-
-    def to_dict(self):
-        return asdict(self)
-
-
-@dataclass
-class Ingredient:
-    rxcui: str | None
-    name: str
-
-
-@dataclass
-class ResolvedMedication:
-    id: str
-    display_name: str
-    ingredients: list
-    resolved_by: str  # ingredients / ndc / rxcui / name / approximate / unresolved
-    matched_name: str | None = None
-    note: str | None = None
-
-
-def resolve_medication(med, rxnorm):
-    """
-    The active ingredients of one medication. med: dict with id and any of name, rxcui, ndc, ingredients (names
-    printed on the label, for medicines without US codes). Order of trust: explicit ingredients, NDC, RxCUI, exact
-    name, then approximate name (reported, so the app can ask "did you mean ...?").
-    """
-    mid, name = str(med["id"]), med.get("name") or ""
-    display = name or ", ".join(med.get("ingredients") or []) or mid
-    try:
-        if med.get("ingredients"):
-            found = []
-            for ing in med["ingredients"]:
-                rxcui = rxnorm.rxcui_by_name(ing) if rxnorm else None
-                ins = rxnorm.ingredients(rxcui) if rxcui else []
-                found += [Ingredient(r, n) for r, n in ins] or [Ingredient(None, normalise_name(ing))]
-            return ResolvedMedication(mid, display, found, "ingredients")
-        if rxnorm is None:
-            return ResolvedMedication(mid, display, [], "unresolved", note="RxNorm lookups are switched off")
-        for how, rxcui in (("ndc", med.get("ndc") and rxnorm.rxcui_by_ndc(med["ndc"])),
-                           ("rxcui", med.get("rxcui")),
-                           ("name", name and rxnorm.rxcui_by_name(name))):
-            if rxcui:
-                ins = rxnorm.ingredients(rxcui)
-                if ins:
-                    if not name:  # entered by code only: show RxNorm's name, not the app's id
-                        display = (rxnorm.properties(rxcui) or {}).get("name") or display
-                    return ResolvedMedication(mid, display, [Ingredient(r, n) for r, n in ins], how)
-        if name:
-            for rxcui, matched in rxnorm.approximate(name):  # best first; skip retired concepts
-                ins = rxnorm.ingredients(rxcui)
-                if ins:
-                    return ResolvedMedication(mid, display, [Ingredient(r, n) for r, n in ins], "approximate",
-                                              matched_name=matched)
-    except Exception as error:  # RxNorm unreachable: report it, don't guess
-        return ResolvedMedication(mid, display, [], "unresolved", note=f"lookup failed: {error}")
-    return ResolvedMedication(mid, display, [], "unresolved", note="no match in RxNorm")
-
-
-def _ingredient_list(names):
-    return names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1]
-
-
-def check_medications(resolved, db):
-    """
-    Interactions and duplications across a patient's medication list (list of ResolvedMedication).
-    Returns (alerts, per_medication), where per_medication says how each one was identified and checked.
-    """
-    alerts, per_med = [], []
-    ddinter = {}  # medication id -> [(ingredient, [ddinter rows])]
-    for m in resolved:
-        rows = [(ing, db.drugs_for_ingredient(ing.rxcui, ing.name)) for ing in m.ingredients]
-        ddinter[m.id] = rows
-        not_listed = [ing.name for ing, r in rows if not r]
-        per_med.append({
-            "id": m.id, "name": m.display_name, "resolved_by": m.resolved_by, "matched_name": m.matched_name,
-            "ingredients": [{"rxcui": i.rxcui, "name": i.name} for i in m.ingredients],
-            "not_in_interaction_database": not_listed, "note": m.note,
-        })
-        if m.resolved_by == "unresolved":
-            alerts.append(Alert(
-                "unresolved", "moderate",
-                f"I couldn't identify {m.display_name}, so it was not checked for interactions. "
-                "Please add its active ingredient from the label.",
-                [m.id], SOURCE_RXNORM, details={"reason": m.note}))
-
-    # therapeutic duplication: the same active ingredient in two medications (brand + generic, or a combination
-    # pill + a single one). Amber, not red: two strengths of one drug can be prescribed together on purpose.
-    for a, b in combinations(resolved, 2):
-        shared = sorted({i.name for i in a.ingredients} & {i.name for i in b.ingredients}
-                        | {i.name for i in a.ingredients for j in b.ingredients if i.rxcui and i.rxcui == j.rxcui})
-        if shared:
-            alerts.append(Alert(
-                "duplication", "moderate",
-                f"{a.display_name} and {b.display_name} both contain {_ingredient_list(shared)}, "
-                "so together they may be a double dose.",
-                [a.id, b.id], SOURCE_RXNORM, details={"shared_ingredients": shared}))
-
-    # interactions: every ingredient of one medication against every ingredient of another
-    for a, b in combinations(resolved, 2):
-        found = []
-        for ing_a, rows_a in ddinter[a.id]:
-            for ing_b, rows_b in ddinter[b.id]:
-                if ing_a.rxcui and ing_a.rxcui == ing_b.rxcui:
-                    continue  # same ingredient: a duplication, handled above
-                for ra in rows_a:
-                    for rb in rows_b:
-                        level = db.level(ra[0], rb[0])
-                        if level:
-                            found.append({"ingredient_a": ing_a.name, "ingredient_b": ing_b.name,
-                                          "ddinter_a": ra[1], "ddinter_b": rb[1], "level": level})
-        if not found:
-            continue
-        severities = [UNKNOWN_LEVEL_AS if f["level"] == "Unknown" else f["level"].lower() for f in found]
-        severity = max(severities, key=RANK.get)
-        graded = severity != "ungraded"
-        words = {"major": "a serious", "moderate": "a moderate", "minor": "a minor", "ungraded": ""}[severity]
-        text = (f"{a.display_name} and {b.display_name} have {words} interaction." if graded else
-                f"{a.display_name} and {b.display_name} are listed as interacting, but the severity isn't graded.")
-        alerts.append(Alert("interaction", severity, text, [a.id, b.id], SOURCE_DDINTER,
-                            severity_graded=graded, details={"pairs": found}))
-
-    alerts.sort(key=lambda al: -RANK[al.severity])
-    return alerts, per_med
-
-
-def _number_word(n):
-    words = ["no", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]
-    return words[n] if 0 <= n < len(words) else str(n)
-
-
-def _count_words(n, singular, plural=None):
-    return f"{_number_word(n)} {singular if n == 1 else (plural or singular + 's')}"
-
-
-def check_dose(schedule, pills, names=None):
-    """
-    Dose check after Stage 2's matching.
-      schedule: [{"medication_id", "count"}] for this dose (count = pills expected)
-      pills:    [{"medication_id", "decision"}], decision = match / unknown / ambiguous, from /v1/match
-      names:    optional {medication_id: display name} for messages (enrolled medications)
-    Returns (alerts, summary dict).
-    """
-    names = {str(k): v for k, v in (names or {}).items()}
-    expected = {str(s["medication_id"]): int(s.get("count", 1)) for s in schedule}
-    matched, unknown, ambiguous = {}, 0, 0
-    for p in pills:
-        decision = p.get("decision", "unknown")
-        if decision == "match" and p.get("medication_id") is not None:
-            mid = str(p["medication_id"])
-            matched[mid] = matched.get(mid, 0) + 1
-        elif decision == "ambiguous":
-            ambiguous += 1
-        else:
-            unknown += 1
-    name = lambda mid: names.get(mid, mid)
-    alerts = []
-    if unknown:
-        what = "One pill is" if unknown == 1 else f"{_count_words(unknown, 'pill').capitalize()} are"
-        alerts.append(Alert("unknown_pill", "major",
-                            f"{what} not one of your medicines. Please set {'it' if unknown == 1 else 'them'} aside.",
-                            [], SOURCE_SCHEDULE, details={"count": unknown}))
-    for mid, n in sorted(matched.items()):
-        e = expected.get(mid, 0)
-        if e == 0:
-            alerts.append(Alert("not_scheduled", "major",
-                                f"{'One pill looks' if n == 1 else _count_words(n, 'pill').capitalize() + ' look'} like "
-                                f"your {name(mid)}, which is not part of this dose. Please set "
-                                f"{'it' if n == 1 else 'them'} aside.",
-                                [mid], SOURCE_SCHEDULE, details={"found": n, "expected": 0}))
-        elif n > e:
-            alerts.append(Alert("double_dose", "major",
-                                f"This dose has {_count_words(n, name(mid) + ' pill')} instead of "
-                                f"{_number_word(e)}. Please set the extra "
-                                f"{'one' if n - e == 1 else 'ones'} aside.",
-                                [mid], SOURCE_SCHEDULE, details={"found": n, "expected": e}))
-    for mid, e in sorted(expected.items()):
-        n = matched.get(mid, 0)
-        if n < e:
-            missing = ("is missing from this dose" if n == 0 else
-                       f"is short: {_count_words(n, 'pill')} instead of {_number_word(e)}")
-            alerts.append(Alert("missed", "moderate", f"Your {name(mid)} {missing}.", [mid], SOURCE_SCHEDULE,
-                                details={"found": n, "expected": e}))
-    if ambiguous:
-        alerts.append(Alert("unsure_pill", "moderate",
-                            f"I'm not sure about {'one pill' if ambiguous == 1 else _count_words(ambiguous, 'pill')}. "
-                            "Please turn it over and photograph the dose again.", [], SOURCE_SCHEDULE,
-                            details={"count": ambiguous}))
-    alerts.sort(key=lambda al: -RANK[al.severity])
-    total = len(pills)
-    summary = {
-        "pills_seen": total, "matched": sum(matched.values()), "unknown": unknown, "unsure": ambiguous,
-        "all_ok": not alerts,
-        "text": (f"I see {_count_words(total, 'pill')}." +
-                 (" They all match this dose." if not alerts and total else "")),
-    }
-    return alerts, summary
+"""
+Stage 3 rules: fixed logic on curated data, no trained model (ROADMAP §4).
+
+  check_medications  when a medication is added or changed: drug-drug interactions (DDInter 2.0) and therapeutic
+                     duplication (two medications sharing an active ingredient, via RxNorm)
+  check_dose         at every dose check: extra pills, pills not part of this dose, scheduled medicines not seen, unknown and
+                     unsure pills (from Stage 2's matching)
+
+Alert rules from the project plan:
+  - Major: red, spoken, caregiver notified. Moderate: amber, spoken. Minor: shown on screen only.
+  - Every alert ends with "Check with your pharmacist." The app never tells anyone to take or stop a medication.
+  - DDInter's "Unknown" level (listed, but not graded) gets its own severity, "ungraded": on screen only, not spoken
+    (decision D6, 2026-10-08), so common medication lists don't drown the graded alerts in noise.
+  - A medication that can't be identified is reported, never silently skipped: no alert must not mean "no
+    interaction" when the check couldn't run.
+"""
+from dataclasses import asdict, dataclass, field
+from itertools import combinations
+
+from epills.core.safety.db import normalise_name
+
+PHARMACIST = "Check with your pharmacist."
+UNKNOWN_LEVEL_AS = "ungraded"  # decision D6
+SOURCE_DDINTER = "DDInter 2.0 (CC BY-NC-SA 4.0), interaction severity"
+SOURCE_RXNORM = "RxNorm (NLM), active ingredients"
+SOURCE_SCHEDULE = "The patient's schedule and the pill check"
+
+STYLE = {  # severity -> how the app shows it (the plan's severity mapping)
+    "major": {"colour": "red", "speak": True, "notify_caregiver": True},
+    "moderate": {"colour": "amber", "speak": True, "notify_caregiver": False},
+    "ungraded": {"colour": "grey", "speak": False, "notify_caregiver": False},
+    "minor": {"colour": "grey", "speak": False, "notify_caregiver": False},
+}
+RANK = {"major": 4, "moderate": 3, "ungraded": 2, "minor": 1}
+
+
+@dataclass
+class Alert:
+    type: str
+    severity: str
+    message: str
+    medication_ids: list
+    source: str
+    severity_graded: bool = True
+    colour: str = ""
+    speak: bool = False
+    notify_caregiver: bool = False
+    details: dict = field(default_factory=dict)
+
+    def __post_init__(self):
+        style = STYLE[self.severity]
+        self.colour, self.speak, self.notify_caregiver = style["colour"], style["speak"], style["notify_caregiver"]
+        if not self.message.endswith(PHARMACIST):
+            self.message = f"{self.message} {PHARMACIST}"
+
+    def to_dict(self):
+        return asdict(self)
+
+
+@dataclass
+class Ingredient:
+    rxcui: str | None
+    name: str
+
+
+@dataclass
+class ResolvedMedication:
+    id: str
+    display_name: str
+    ingredients: list
+    resolved_by: str  # ingredients / ndc / rxcui / name / approximate / unresolved
+    matched_name: str | None = None
+    note: str | None = None
+
+
+def resolve_medication(med, rxnorm):
+    """
+    The active ingredients of one medication. med: dict with id and any of name, rxcui, ndc, ingredients (names
+    printed on the label, for medicines without US codes). Order of trust: explicit ingredients, NDC, RxCUI, exact
+    name, then approximate name (reported, so the app can ask "did you mean ...?").
+    """
+    mid, name = str(med["id"]), med.get("name") or ""
+    display = name or ", ".join(med.get("ingredients") or []) or mid
+    try:
+        if med.get("ingredients"):
+            found = []
+            for ing in med["ingredients"]:
+                rxcui = rxnorm.rxcui_by_name(ing) if rxnorm else None
+                ins = rxnorm.ingredients(rxcui) if rxcui else []
+                found += [Ingredient(r, n) for r, n in ins] or [Ingredient(None, normalise_name(ing))]
+            return ResolvedMedication(mid, display, found, "ingredients")
+        if rxnorm is None:
+            return ResolvedMedication(mid, display, [], "unresolved", note="RxNorm lookups are switched off")
+        for how, rxcui in (("ndc", med.get("ndc") and rxnorm.rxcui_by_ndc(med["ndc"])),
+                           ("rxcui", med.get("rxcui")),
+                           ("name", name and rxnorm.rxcui_by_name(name))):
+            if rxcui:
+                ins = rxnorm.ingredients(rxcui)
+                if ins:
+                    if not name:  # entered by code only: show RxNorm's name, not the app's id
+                        display = (rxnorm.properties(rxcui) or {}).get("name") or display
+                    return ResolvedMedication(mid, display, [Ingredient(r, n) for r, n in ins], how)
+        if name:
+            for rxcui, matched in rxnorm.approximate(name):  # best first; skip retired concepts
+                ins = rxnorm.ingredients(rxcui)
+                if ins:
+                    return ResolvedMedication(mid, display, [Ingredient(r, n) for r, n in ins], "approximate",
+                                              matched_name=matched)
+    except Exception as error:  # RxNorm unreachable: report it, don't guess
+        return ResolvedMedication(mid, display, [], "unresolved", note=f"lookup failed: {error}")
+    return ResolvedMedication(mid, display, [], "unresolved", note="no match in RxNorm")
+
+
+def _ingredient_list(names):
+    return names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1]
+
+
+def check_medications(resolved, db):
+    """
+    Interactions and duplications across a patient's medication list (list of ResolvedMedication).
+    Returns (alerts, per_medication), where per_medication says how each one was identified and checked.
+    """
+    alerts, per_med = [], []
+    ddinter = {}  # medication id -> [(ingredient, [ddinter rows])]
+    for m in resolved:
+        rows = [(ing, db.drugs_for_ingredient(ing.rxcui, ing.name)) for ing in m.ingredients]
+        ddinter[m.id] = rows
+        not_listed = [ing.name for ing, r in rows if not r]
+        per_med.append({
+            "id": m.id, "name": m.display_name, "resolved_by": m.resolved_by, "matched_name": m.matched_name,
+            "ingredients": [{"rxcui": i.rxcui, "name": i.name} for i in m.ingredients],
+            "not_in_interaction_database": not_listed, "note": m.note,
+        })
+        if m.resolved_by == "unresolved":
+            alerts.append(Alert(
+                "unresolved", "moderate",
+                f"I couldn't identify {m.display_name}, so it was not checked for interactions. "
+                "Please add its active ingredient from the label.",
+                [m.id], SOURCE_RXNORM, details={"reason": m.note}))
+
+    # therapeutic duplication: the same active ingredient in two medications (brand + generic, or a combination
+    # pill + a single one). Amber, not red: two strengths of one drug can be prescribed together on purpose.
+    for a, b in combinations(resolved, 2):
+        shared = sorted({i.name for i in a.ingredients} & {i.name for i in b.ingredients}
+                        | {i.name for i in a.ingredients for j in b.ingredients if i.rxcui and i.rxcui == j.rxcui})
+        if shared:
+            alerts.append(Alert(
+                "duplication", "moderate",
+                f"{a.display_name} and {b.display_name} both contain {_ingredient_list(shared)}, "
+                "so together they may be a double dose.",
+                [a.id, b.id], SOURCE_RXNORM, details={"shared_ingredients": shared}))
+
+    # interactions: every ingredient of one medication against every ingredient of another
+    for a, b in combinations(resolved, 2):
+        found = []
+        for ing_a, rows_a in ddinter[a.id]:
+            for ing_b, rows_b in ddinter[b.id]:
+                if ing_a.rxcui and ing_a.rxcui == ing_b.rxcui:
+                    continue  # same ingredient: a duplication, handled above
+                for ra in rows_a:
+                    for rb in rows_b:
+                        level = db.level(ra[0], rb[0])
+                        if level:
+                            found.append({"ingredient_a": ing_a.name, "ingredient_b": ing_b.name,
+                                          "ddinter_a": ra[1], "ddinter_b": rb[1], "level": level})
+        if not found:
+            continue
+        severities = [UNKNOWN_LEVEL_AS if f["level"] == "Unknown" else f["level"].lower() for f in found]
+        severity = max(severities, key=RANK.get)
+        graded = severity != "ungraded"
+        words = {"major": "a serious", "moderate": "a moderate", "minor": "a minor", "ungraded": ""}[severity]
+        text = (f"{a.display_name} and {b.display_name} have {words} interaction." if graded else
+                f"{a.display_name} and {b.display_name} are listed as interacting, but the severity isn't graded.")
+        alerts.append(Alert("interaction", severity, text, [a.id, b.id], SOURCE_DDINTER,
+                            severity_graded=graded, details={"pairs": found}))
+
+    alerts.sort(key=lambda al: -RANK[al.severity])
+    return alerts, per_med
+
+
+def _number_word(n):
+    words = ["no", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]
+    return words[n] if 0 <= n < len(words) else str(n)
+
+
+def _count_words(n, singular, plural=None):
+    return f"{_number_word(n)} {singular if n == 1 else (plural or singular + 's')}"
+
+
+def check_dose(schedule, pills, names=None):
+    """
+    Dose check after Stage 2's matching.
+      schedule: [{"medication_id", "count"}] for this dose (count = pills expected)
+      pills:    [{"medication_id", "decision"}], decision = match / unknown / ambiguous, from /v1/match
+      names:    optional {medication_id: display name} for messages (enrolled medications)
+    Returns (alerts, summary dict).
+    """
+    names = {str(k): v for k, v in (names or {}).items()}
+    expected = {str(s["medication_id"]): int(s.get("count", 1)) for s in schedule}
+    matched, unknown, ambiguous = {}, 0, 0
+    for p in pills:
+        decision = p.get("decision", "unknown")
+        if decision == "match" and p.get("medication_id") is not None:
+            mid = str(p["medication_id"])
+            matched[mid] = matched.get(mid, 0) + 1
+        elif decision == "ambiguous":
+            ambiguous += 1
+        else:
+            unknown += 1
+    name = lambda mid: names.get(mid, mid)
+    alerts = []
+    if unknown:
+        what = "One pill doesn't" if unknown == 1 else f"{_count_words(unknown, 'pill').capitalize()} don't"
+        alerts.append(Alert("unknown_pill", "major",
+                            f"{what} look like any of your medicines. Please set {'it' if unknown == 1 else 'them'} "
+                            "aside.",
+                            [], SOURCE_SCHEDULE, details={"count": unknown}))
+    for mid, n in sorted(matched.items()):
+        e = expected.get(mid, 0)
+        if e == 0:
+            alerts.append(Alert("not_scheduled", "major",
+                                f"{'One pill looks' if n == 1 else _count_words(n, 'pill').capitalize() + ' look'} like "
+                                f"your {name(mid)}, which is not part of this dose. Please set "
+                                f"{'it' if n == 1 else 'them'} aside.",
+                                [mid], SOURCE_SCHEDULE, details={"found": n, "expected": 0}))
+        elif n > e:
+            alerts.append(Alert("double_dose", "major",
+                                f"This photo shows {_count_words(n, 'pill')} that look like your {name(mid)} instead "
+                                f"of {_number_word(e)}. Please set the extra {'one' if n - e == 1 else 'ones'} aside.",
+                                [mid], SOURCE_SCHEDULE, details={"found": n, "expected": e}))
+    for mid, e in sorted(expected.items()):
+        n = matched.get(mid, 0)
+        if n < e:  # not seen in this photo: never a missed dose (the backend knows what was taken)
+            text = (f"I can't see your {name(mid)} in this photo." if n == 0 else
+                    f"I can see only {_count_words(n, 'pill')} that look like your {name(mid)}, instead of "
+                    f"{_number_word(e)}.")
+            alerts.append(Alert("not_seen", "moderate", text, [mid], SOURCE_SCHEDULE,
+                                details={"found": n, "expected": e}))
+    if ambiguous:
+        alerts.append(Alert("unsure_pill", "moderate",
+                            f"I'm not sure about {'one pill' if ambiguous == 1 else _count_words(ambiguous, 'pill')}. "
+                            "Please turn it over and photograph the dose again.", [], SOURCE_SCHEDULE,
+                            details={"count": ambiguous}))
+    alerts.sort(key=lambda al: -RANK[al.severity])
+    total = len(pills)
+    summary = {
+        "pills_seen": total, "matched": sum(matched.values()), "unknown": unknown, "unsure": ambiguous,
+        "all_ok": not alerts,  # deprecated name of all_visually_matched
+        "all_visually_matched": not alerts,
+        "count_reliable": True,
+        "text": (f"I see {_count_words(total, 'pill')}." +
+                 (" They all look like the medicines of this dose." if not alerts and total else "")),
+    }
+    return alerts, summary