Spaces:
Sleeping
Sleeping
File size: 3,779 Bytes
087643a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | # Decides if the evidence is good enough to answer confidently - before the
# LLM writes anything. Kept as a separate deterministic check because asking
# a model "are you confident?" just gets more text, not a real signal - it's
# equally fluent at a confident-sounding guess as an honest refusal.
from __future__ import annotations
from dataclasses import dataclass, field
from app.reasoning.evidence import Evidence
# Below this, a single detection isn't trustworthy enough to answer from.
# 0.50 rather than something stricter like 0.7 - correctly localised
# regions on this dataset tend to score well above 0.6 in my own runs.
CONFIDENCE_FLOOR = 0.50
# For counting: don't report an exact number if too much of it sits in this
# shaky band. Below 0.25 is just noise; 0.25-0.50 is "plausible but I
# wouldn't bet on the count."
AMBIGUOUS_BAND = (0.25, CONFIDENCE_FLOOR)
AMBIGUOUS_BAND_FRACTION = 0.30
@dataclass
class GuardrailVerdict:
triggered: bool
rule: str | None = None
detail: dict = field(default_factory=dict)
def _confidences_for(evidence: Evidence, target_classes: list[str]) -> list[float]:
"""Max confidence per target class, ignoring unrelated classes on the page."""
values: list[float] = []
for class_name in target_classes:
stats = evidence.confidence_stats.get(class_name)
if stats is None:
continue
values.append(stats.max)
return values
def evaluate_guardrail(
evidence: Evidence,
target_classes: list[str],
task_type: str,
saturated: bool = False,
) -> GuardrailVerdict:
"""Four rules, checked in order, most specific first - some can overlap
and I want the more useful explanation to win (e.g. "found nothing"
beats "page was saturated" even when both are technically true)."""
raw_confidences = [
value
for class_name in target_classes
for value in evidence.confidences_by_class.get(class_name, [])
]
target_counts = {
class_name: evidence.class_counts.get(class_name, 0)
for class_name in target_classes
}
total_target_detections = sum(target_counts.values())
# nothing of the target class found at all
if total_target_detections == 0:
return GuardrailVerdict(
triggered=True,
rule="no_detections",
detail={"target_classes": target_classes},
)
max_confidences = _confidences_for(evidence, target_classes)
observed_max = max(max_confidences) if max_confidences else 0.0
# found something, but nothing clears the confidence floor
if observed_max < CONFIDENCE_FLOOR:
return GuardrailVerdict(
triggered=True,
rule="all_below_tau",
detail={"tau": CONFIDENCE_FLOOR, "observed_max": observed_max},
)
# counting only: too much of the count sits in the shaky band.
# doesn't apply to "is there one?" - one confident hit is enough for that.
if task_type == "count" and raw_confidences:
low, high = AMBIGUOUS_BAND
in_band = sum(1 for c in raw_confidences if low <= c < high)
if raw_confidences and (in_band / len(raw_confidences)) > AMBIGUOUS_BAND_FRACTION:
return GuardrailVerdict(
triggered=True,
rule="ambiguous_band",
detail={
"band": list(AMBIGUOUS_BAND),
"fraction_in_band": round(in_band / len(raw_confidences), 2),
},
)
# page had more regions than RT-DETR's query budget could emit -
# a count here is unreliable regardless of confidence
if saturated:
return GuardrailVerdict(triggered=True, rule="query_saturated", detail={})
return GuardrailVerdict(triggered=False)
|