Spaces:
Sleeping
Sleeping
File size: 2,500 Bytes
c2b7494 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | """Select uncertain classifications requiring targeted human review."""
from __future__ import annotations
from dataclasses import dataclass
from src.llm.classify import ClassifiedCandidate
@dataclass(slots=True)
class ReviewItem:
code: str
description: str
vocabulary: str
label: str
confidence: float
explanation: str
reason: str
def select_uncertain(
classified: list[ClassifiedCandidate],
confidence_threshold: float = 0.7,
possible_requires_review: bool = False,
) -> list[ReviewItem]:
"""Select classified candidates that require human validation.
A candidate is flagged for review when:
- its LLM confidence is below *confidence_threshold*, or
- it was classified as 'Possible' and *possible_requires_review* is True.
Parameters
----------
classified : list[ClassifiedCandidate]
Output of :func:`src.llm.classify.llm_classify`.
confidence_threshold : float
Minimum confidence below which a candidate is routed for review.
possible_requires_review : bool
Whether 'Possible' classifications are always routed for review,
regardless of confidence.
Returns
-------
list[ReviewItem]
Items requiring human review, each annotated with the reason.
"""
review_items: list[ReviewItem] = []
for c in classified:
reasons: list[str] = []
if c.decision_origin == "codemapper":
reasons.append("included by CodeMapper mapping; EDF scope not independently assessed")
if c.confidence < confidence_threshold:
reasons.append(f"confidence {c.confidence:.2f} below threshold {confidence_threshold:.2f}")
if possible_requires_review and c.label == "Possible":
reasons.append("classified as 'Possible'")
if reasons:
review_items.append(
ReviewItem(
code=c.code,
description=c.description,
vocabulary=c.vocabulary,
label=c.label,
confidence=c.confidence,
explanation=c.explanation,
reason="; ".join(reasons),
)
)
return review_items
def review_rate(classified: list[ClassifiedCandidate], review_items: list[ReviewItem]) -> float:
"""Fraction of classified candidates flagged for human review."""
if not classified:
return 0.0
return len(review_items) / len(classified)
|