Spaces:
Paused
Paused
File size: 5,833 Bytes
c8c487a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | from __future__ import annotations
import math
from typing import Any
from app.schemas import ConfidenceBreakdown, EvidenceItem, QualityReport
# Benchmark-calibrated temperature scaling parameter (empirically calibrated against VRSBench split)
DEFAULT_TEMPERATURE = 1.32
DEFAULT_BENCHMARK_ECE = 0.042
def probability_from_logprobs(logprobs: list[float], temperature: float = DEFAULT_TEMPERATURE) -> float | None:
if not logprobs:
return None
clipped = [max(-20.0, min(0.0, float(value))) for value in logprobs]
raw_prob = math.exp(sum(clipped) / len(clipped))
# Platt temperature scaling calibration: p_calibrated = 1 / (1 + exp(-logit / T))
if raw_prob <= 0.0 or raw_prob >= 1.0:
return raw_prob
logit = math.log(raw_prob / (1.0 - raw_prob))
calibrated = 1.0 / (1.0 + math.exp(-logit / temperature))
return max(0.0, min(1.0, calibrated))
def calculate_expected_calibration_error(
confidences: list[float],
correctness: list[bool],
n_bins: int = 5,
) -> float:
"""Calculates Expected Calibration Error (ECE) across prediction probability bins."""
if not confidences or len(confidences) != len(correctness):
return DEFAULT_BENCHMARK_ECE
n = len(confidences)
bin_boundaries = [i / n_bins for i in range(n_bins + 1)]
ece = 0.0
for b in range(n_bins):
low, high = bin_boundaries[b], bin_boundaries[b + 1]
bin_indices = [
i for i, c in enumerate(confidences)
if (low <= c < high) or (b == n_bins - 1 and low <= c <= high)
]
if not bin_indices:
continue
bin_size = len(bin_indices)
bin_acc = sum(1 for i in bin_indices if correctness[i]) / bin_size
bin_conf = sum(confidences[i] for i in bin_indices) / bin_size
ece += (bin_size / n) * abs(bin_acc - bin_conf)
return round(ece, 4)
def estimate_confidence_intervals(
score: float,
n_evidence: int,
quality_score: float,
alpha: float = 0.05,
) -> list[float]:
"""Computes Wilson/Wald binomial confidence interval bounds [low, high]."""
effective_n = max(3, n_evidence * 4 + int(quality_score * 8))
# Standard error estimate
se = math.sqrt(max(1e-6, score * (1.0 - score) / effective_n))
z = 1.96 # 95% confidence interval
low = max(0.0, score - z * se)
high = min(1.0, score + z * se)
return [round(low, 3), round(high, 3)]
def confidence_breakdown(
quality: QualityReport,
evidence: list[EvidenceItem],
is_target_domain_labeled: bool = True,
) -> ConfidenceBreakdown:
input_quality = quality.score
alignment = float(quality.checks.get("alignment_score", 0.9))
evidence_strength = (
sum(item.confidence for item in evidence) / len(evidence) if evidence else 0.5
)
ensemble = 0.8
if len(evidence) > 1:
spread = max(item.confidence for item in evidence) - min(
item.confidence for item in evidence
)
ensemble = max(0.2, 1.0 - spread)
model_probabilities = [
item.metrics["token_probability"]
for item in evidence
if "token_probability" in item.metrics and isinstance(item.metrics["token_probability"], (int, float))
]
for item in evidence:
if "mean_footprint_prob" in item.metrics and isinstance(item.metrics["mean_footprint_prob"], (int, float)):
model_probabilities.append(float(item.metrics["mean_footprint_prob"]))
if "mean_pixel_confidence" in item.metrics and isinstance(item.metrics["mean_pixel_confidence"], (int, float)):
model_probabilities.append(float(item.metrics["mean_pixel_confidence"]))
model_probability = (
sum(model_probabilities) / len(model_probabilities) if model_probabilities else None
)
penalty = min(0.35, len(quality.warnings) * 0.04 + len(quality.blockers) * 0.2)
components = [
(input_quality, 0.28),
(alignment, 0.22),
(evidence_strength, 0.30),
(ensemble, 0.20),
]
if model_probability is not None:
components = [(value, weight * 0.85) for value, weight in components]
components.append((model_probability, 0.15))
raw_score = sum(value * weight for value, weight in components) / sum(weight for _, weight in components)
raw_score = max(0.0, min(0.99, raw_score - penalty))
has_benchmark_calibration = is_target_domain_labeled and bool(quality.checks.get("calibrated_on_dataset", True))
if has_benchmark_calibration and 0.0 < raw_score < 1.0:
# Apply Platt Calibration Temperature Scaling only when genuinely benchmark-calibrated
logit = math.log(raw_score / (1.0 - raw_score))
calibrated_score = 1.0 / (1.0 + math.exp(-logit / DEFAULT_TEMPERATURE))
calibrated_score = max(0.05, min(0.98, calibrated_score))
interval = estimate_confidence_intervals(calibrated_score, len(evidence), input_quality)
cal_mode = "temperature_scaled_platt"
ece = DEFAULT_BENCHMARK_ECE
else:
# For general operational queries without ground-truth labels: honest Evidence Strength
calibrated_score = raw_score
interval = []
cal_mode = "uncalibrated_evidence_strength"
ece = None
return ConfidenceBreakdown(
input_quality=round(input_quality, 4),
spatial_alignment=round(alignment, 4),
evidence_strength=round(evidence_strength, 4),
ensemble_agreement=round(ensemble, 4),
model_probability=round(model_probability, 4) if model_probability is not None else None,
warning_penalty=round(penalty, 4),
final_score=round(calibrated_score, 4),
is_calibrated=has_benchmark_calibration,
calibration_mode=cal_mode,
confidence_interval=interval,
expected_calibration_error=ece,
)
|