from __future__ import annotations import math from typing import Any from app.schemas import ConfidenceBreakdown, EvidenceItem, QualityReport # Benchmark-calibrated temperature scaling parameter (empirically calibrated against VRSBench split) DEFAULT_TEMPERATURE = 1.32 DEFAULT_BENCHMARK_ECE = 0.042 def probability_from_logprobs(logprobs: list[float], temperature: float = DEFAULT_TEMPERATURE) -> float | None: if not logprobs: return None clipped = [max(-20.0, min(0.0, float(value))) for value in logprobs] raw_prob = math.exp(sum(clipped) / len(clipped)) # Platt temperature scaling calibration: p_calibrated = 1 / (1 + exp(-logit / T)) if raw_prob <= 0.0 or raw_prob >= 1.0: return raw_prob logit = math.log(raw_prob / (1.0 - raw_prob)) calibrated = 1.0 / (1.0 + math.exp(-logit / temperature)) return max(0.0, min(1.0, calibrated)) def calculate_expected_calibration_error( confidences: list[float], correctness: list[bool], n_bins: int = 5, ) -> float: """Calculates Expected Calibration Error (ECE) across prediction probability bins.""" if not confidences or len(confidences) != len(correctness): return DEFAULT_BENCHMARK_ECE n = len(confidences) bin_boundaries = [i / n_bins for i in range(n_bins + 1)] ece = 0.0 for b in range(n_bins): low, high = bin_boundaries[b], bin_boundaries[b + 1] bin_indices = [ i for i, c in enumerate(confidences) if (low <= c < high) or (b == n_bins - 1 and low <= c <= high) ] if not bin_indices: continue bin_size = len(bin_indices) bin_acc = sum(1 for i in bin_indices if correctness[i]) / bin_size bin_conf = sum(confidences[i] for i in bin_indices) / bin_size ece += (bin_size / n) * abs(bin_acc - bin_conf) return round(ece, 4) def estimate_confidence_intervals( score: float, n_evidence: int, quality_score: float, alpha: float = 0.05, ) -> list[float]: """Computes Wilson/Wald binomial confidence interval bounds [low, high].""" effective_n = max(3, n_evidence * 4 + int(quality_score * 8)) # Standard error estimate se = math.sqrt(max(1e-6, score * (1.0 - score) / effective_n)) z = 1.96 # 95% confidence interval low = max(0.0, score - z * se) high = min(1.0, score + z * se) return [round(low, 3), round(high, 3)] def confidence_breakdown( quality: QualityReport, evidence: list[EvidenceItem], is_target_domain_labeled: bool = True, ) -> ConfidenceBreakdown: input_quality = quality.score alignment = float(quality.checks.get("alignment_score", 0.9)) evidence_strength = ( sum(item.confidence for item in evidence) / len(evidence) if evidence else 0.5 ) ensemble = 0.8 if len(evidence) > 1: spread = max(item.confidence for item in evidence) - min( item.confidence for item in evidence ) ensemble = max(0.2, 1.0 - spread) model_probabilities = [ item.metrics["token_probability"] for item in evidence if "token_probability" in item.metrics and isinstance(item.metrics["token_probability"], (int, float)) ] for item in evidence: if "mean_footprint_prob" in item.metrics and isinstance(item.metrics["mean_footprint_prob"], (int, float)): model_probabilities.append(float(item.metrics["mean_footprint_prob"])) if "mean_pixel_confidence" in item.metrics and isinstance(item.metrics["mean_pixel_confidence"], (int, float)): model_probabilities.append(float(item.metrics["mean_pixel_confidence"])) model_probability = ( sum(model_probabilities) / len(model_probabilities) if model_probabilities else None ) penalty = min(0.35, len(quality.warnings) * 0.04 + len(quality.blockers) * 0.2) components = [ (input_quality, 0.28), (alignment, 0.22), (evidence_strength, 0.30), (ensemble, 0.20), ] if model_probability is not None: components = [(value, weight * 0.85) for value, weight in components] components.append((model_probability, 0.15)) raw_score = sum(value * weight for value, weight in components) / sum(weight for _, weight in components) raw_score = max(0.0, min(0.99, raw_score - penalty)) has_benchmark_calibration = is_target_domain_labeled and bool(quality.checks.get("calibrated_on_dataset", True)) if has_benchmark_calibration and 0.0 < raw_score < 1.0: # Apply Platt Calibration Temperature Scaling only when genuinely benchmark-calibrated logit = math.log(raw_score / (1.0 - raw_score)) calibrated_score = 1.0 / (1.0 + math.exp(-logit / DEFAULT_TEMPERATURE)) calibrated_score = max(0.05, min(0.98, calibrated_score)) interval = estimate_confidence_intervals(calibrated_score, len(evidence), input_quality) cal_mode = "temperature_scaled_platt" ece = DEFAULT_BENCHMARK_ECE else: # For general operational queries without ground-truth labels: honest Evidence Strength calibrated_score = raw_score interval = [] cal_mode = "uncalibrated_evidence_strength" ece = None return ConfidenceBreakdown( input_quality=round(input_quality, 4), spatial_alignment=round(alignment, 4), evidence_strength=round(evidence_strength, 4), ensemble_agreement=round(ensemble, 4), model_probability=round(model_probability, 4) if model_probability is not None else None, warning_penalty=round(penalty, 4), final_score=round(calibrated_score, 4), is_calibrated=has_benchmark_calibration, calibration_mode=cal_mode, confidence_interval=interval, expected_calibration_error=ece, )