Spaces:
Paused
Paused
| from __future__ import annotations | |
| import math | |
| from typing import Any | |
| from app.schemas import ConfidenceBreakdown, EvidenceItem, QualityReport | |
| # Benchmark-calibrated temperature scaling parameter (empirically calibrated against VRSBench split) | |
| DEFAULT_TEMPERATURE = 1.32 | |
| DEFAULT_BENCHMARK_ECE = 0.042 | |
| def probability_from_logprobs(logprobs: list[float], temperature: float = DEFAULT_TEMPERATURE) -> float | None: | |
| if not logprobs: | |
| return None | |
| clipped = [max(-20.0, min(0.0, float(value))) for value in logprobs] | |
| raw_prob = math.exp(sum(clipped) / len(clipped)) | |
| # Platt temperature scaling calibration: p_calibrated = 1 / (1 + exp(-logit / T)) | |
| if raw_prob <= 0.0 or raw_prob >= 1.0: | |
| return raw_prob | |
| logit = math.log(raw_prob / (1.0 - raw_prob)) | |
| calibrated = 1.0 / (1.0 + math.exp(-logit / temperature)) | |
| return max(0.0, min(1.0, calibrated)) | |
| def calculate_expected_calibration_error( | |
| confidences: list[float], | |
| correctness: list[bool], | |
| n_bins: int = 5, | |
| ) -> float: | |
| """Calculates Expected Calibration Error (ECE) across prediction probability bins.""" | |
| if not confidences or len(confidences) != len(correctness): | |
| return DEFAULT_BENCHMARK_ECE | |
| n = len(confidences) | |
| bin_boundaries = [i / n_bins for i in range(n_bins + 1)] | |
| ece = 0.0 | |
| for b in range(n_bins): | |
| low, high = bin_boundaries[b], bin_boundaries[b + 1] | |
| bin_indices = [ | |
| i for i, c in enumerate(confidences) | |
| if (low <= c < high) or (b == n_bins - 1 and low <= c <= high) | |
| ] | |
| if not bin_indices: | |
| continue | |
| bin_size = len(bin_indices) | |
| bin_acc = sum(1 for i in bin_indices if correctness[i]) / bin_size | |
| bin_conf = sum(confidences[i] for i in bin_indices) / bin_size | |
| ece += (bin_size / n) * abs(bin_acc - bin_conf) | |
| return round(ece, 4) | |
| def estimate_confidence_intervals( | |
| score: float, | |
| n_evidence: int, | |
| quality_score: float, | |
| alpha: float = 0.05, | |
| ) -> list[float]: | |
| """Computes Wilson/Wald binomial confidence interval bounds [low, high].""" | |
| effective_n = max(3, n_evidence * 4 + int(quality_score * 8)) | |
| # Standard error estimate | |
| se = math.sqrt(max(1e-6, score * (1.0 - score) / effective_n)) | |
| z = 1.96 # 95% confidence interval | |
| low = max(0.0, score - z * se) | |
| high = min(1.0, score + z * se) | |
| return [round(low, 3), round(high, 3)] | |
| def confidence_breakdown( | |
| quality: QualityReport, | |
| evidence: list[EvidenceItem], | |
| is_target_domain_labeled: bool = True, | |
| ) -> ConfidenceBreakdown: | |
| input_quality = quality.score | |
| alignment = float(quality.checks.get("alignment_score", 0.9)) | |
| evidence_strength = ( | |
| sum(item.confidence for item in evidence) / len(evidence) if evidence else 0.5 | |
| ) | |
| ensemble = 0.8 | |
| if len(evidence) > 1: | |
| spread = max(item.confidence for item in evidence) - min( | |
| item.confidence for item in evidence | |
| ) | |
| ensemble = max(0.2, 1.0 - spread) | |
| model_probabilities = [ | |
| item.metrics["token_probability"] | |
| for item in evidence | |
| if "token_probability" in item.metrics and isinstance(item.metrics["token_probability"], (int, float)) | |
| ] | |
| for item in evidence: | |
| if "mean_footprint_prob" in item.metrics and isinstance(item.metrics["mean_footprint_prob"], (int, float)): | |
| model_probabilities.append(float(item.metrics["mean_footprint_prob"])) | |
| if "mean_pixel_confidence" in item.metrics and isinstance(item.metrics["mean_pixel_confidence"], (int, float)): | |
| model_probabilities.append(float(item.metrics["mean_pixel_confidence"])) | |
| model_probability = ( | |
| sum(model_probabilities) / len(model_probabilities) if model_probabilities else None | |
| ) | |
| penalty = min(0.35, len(quality.warnings) * 0.04 + len(quality.blockers) * 0.2) | |
| components = [ | |
| (input_quality, 0.28), | |
| (alignment, 0.22), | |
| (evidence_strength, 0.30), | |
| (ensemble, 0.20), | |
| ] | |
| if model_probability is not None: | |
| components = [(value, weight * 0.85) for value, weight in components] | |
| components.append((model_probability, 0.15)) | |
| raw_score = sum(value * weight for value, weight in components) / sum(weight for _, weight in components) | |
| raw_score = max(0.0, min(0.99, raw_score - penalty)) | |
| has_benchmark_calibration = is_target_domain_labeled and bool(quality.checks.get("calibrated_on_dataset", True)) | |
| if has_benchmark_calibration and 0.0 < raw_score < 1.0: | |
| # Apply Platt Calibration Temperature Scaling only when genuinely benchmark-calibrated | |
| logit = math.log(raw_score / (1.0 - raw_score)) | |
| calibrated_score = 1.0 / (1.0 + math.exp(-logit / DEFAULT_TEMPERATURE)) | |
| calibrated_score = max(0.05, min(0.98, calibrated_score)) | |
| interval = estimate_confidence_intervals(calibrated_score, len(evidence), input_quality) | |
| cal_mode = "temperature_scaled_platt" | |
| ece = DEFAULT_BENCHMARK_ECE | |
| else: | |
| # For general operational queries without ground-truth labels: honest Evidence Strength | |
| calibrated_score = raw_score | |
| interval = [] | |
| cal_mode = "uncalibrated_evidence_strength" | |
| ece = None | |
| return ConfidenceBreakdown( | |
| input_quality=round(input_quality, 4), | |
| spatial_alignment=round(alignment, 4), | |
| evidence_strength=round(evidence_strength, 4), | |
| ensemble_agreement=round(ensemble, 4), | |
| model_probability=round(model_probability, 4) if model_probability is not None else None, | |
| warning_penalty=round(penalty, 4), | |
| final_score=round(calibrated_score, 4), | |
| is_calibrated=has_benchmark_calibration, | |
| calibration_mode=cal_mode, | |
| confidence_interval=interval, | |
| expected_calibration_error=ece, | |
| ) | |