Spaces:
Paused
Paused
| from __future__ import annotations | |
| import math | |
| from typing import Any | |
| from satquery_engine.schemas import ConfidenceBreakdown, EvidenceItem, QualityReport | |
| # No fitted calibration data is installed. | |
| DEFAULT_TEMPERATURE = 1.0 | |
| DEFAULT_BENCHMARK_ECE = None | |
| def probability_from_logprobs(logprobs: list[float], temperature: float = DEFAULT_TEMPERATURE) -> float | None: | |
| if not logprobs: | |
| return None | |
| clipped = [max(-20.0, min(0.0, float(value))) for value in logprobs] | |
| raw_prob = math.exp(sum(clipped) / len(clipped)) | |
| # Platt temperature scaling calibration: p_calibrated = 1 / (1 + exp(-logit / T)) | |
| if raw_prob <= 0.0 or raw_prob >= 1.0: | |
| return raw_prob | |
| logit = math.log(raw_prob / (1.0 - raw_prob)) | |
| calibrated = 1.0 / (1.0 + math.exp(-logit / temperature)) | |
| return max(0.0, min(1.0, calibrated)) | |
| def calculate_expected_calibration_error( | |
| confidences: list[float], | |
| correctness: list[bool], | |
| n_bins: int = 5, | |
| ) -> float: | |
| """Calculates Expected Calibration Error (ECE) across prediction probability bins.""" | |
| if not confidences or len(confidences) != len(correctness): | |
| return DEFAULT_BENCHMARK_ECE | |
| n = len(confidences) | |
| bin_boundaries = [i / n_bins for i in range(n_bins + 1)] | |
| ece = 0.0 | |
| for b in range(n_bins): | |
| low, high = bin_boundaries[b], bin_boundaries[b + 1] | |
| bin_indices = [ | |
| i for i, c in enumerate(confidences) | |
| if (low <= c < high) or (b == n_bins - 1 and low <= c <= high) | |
| ] | |
| if not bin_indices: | |
| continue | |
| bin_size = len(bin_indices) | |
| bin_acc = sum(1 for i in bin_indices if correctness[i]) / bin_size | |
| bin_conf = sum(confidences[i] for i in bin_indices) / bin_size | |
| ece += (bin_size / n) * abs(bin_acc - bin_conf) | |
| return round(ece, 4) | |
| def estimate_confidence_intervals( | |
| score: float, | |
| n_evidence: int, | |
| quality_score: float, | |
| alpha: float = 0.05, | |
| ) -> list[float]: | |
| """Computes Wilson/Wald binomial confidence interval bounds [low, high].""" | |
| effective_n = max(3, n_evidence * 4 + int(quality_score * 8)) | |
| # Standard error estimate | |
| se = math.sqrt(max(1e-6, score * (1.0 - score) / effective_n)) | |
| z = 1.96 # 95% confidence interval | |
| low = max(0.0, score - z * se) | |
| high = min(1.0, score + z * se) | |
| return [round(low, 3), round(high, 3)] | |
| def confidence_breakdown(quality: QualityReport, evidence: list[EvidenceItem]) -> ConfidenceBreakdown: | |
| # A transparent evidence-strength heuristic, not a calibrated probability. | |
| scores=[] | |
| for item in evidence: | |
| value=item.confidence | |
| if item.kind=="vlm": value=0.0 | |
| if item.kind=="buildings" and (item.metrics.get("benchmark_f1") is None or item.metrics["benchmark_f1"]<.6): | |
| value=min(value,.35) | |
| if "proxy" in str(item.metrics.get("method","")).lower(): value=min(value,.25) | |
| scores.append(value) | |
| strength = min(scores) if scores else 0.0 | |
| alignment = float(quality.checks.get("alignment_score", 1.0)) | |
| penalty = min(.5, .04 * len(quality.warnings)) | |
| score = strength * quality.score * alignment * (1-penalty) if quality.compatible else 0.0 | |
| return ConfidenceBreakdown(input_quality=quality.score, spatial_alignment=alignment, | |
| evidence_strength=strength, ensemble_agreement=0.0, warning_penalty=penalty, | |
| final_score=round(max(0.,min(1.,score)),4), is_calibrated=False, | |
| calibration_mode="weakest_evidence_with_domain_caps_times_quality_alignment_warning_v2", | |
| confidence_interval=[], expected_calibration_error=None) | |