Collaborative-Fraud-Intelligence-Simulator / frontend /src /components /dashboard /CalibrationReliabilityPlot.tsx
| import React, { useState, useMemo } from 'react'; | |
| import { | |
| ResponsiveContainer, | |
| ComposedChart, | |
| Line, | |
| Bar, | |
| XAxis, | |
| YAxis, | |
| CartesianGrid, | |
| Tooltip, | |
| Legend, | |
| } from 'recharts'; | |
| import { Sliders, ShieldCheck, AlertCircle, BarChart3, Info } from 'lucide-react'; | |
| import type { CalibrationReport, CalibrationBinItem } from '../../api/types'; | |
| export interface CalibrationReliabilityPlotProps { | |
| report?: CalibrationReport | null; | |
| isLoading?: boolean; | |
| className?: string; | |
| } | |
| export type CalibrationMethod = 'raw' | 'platt' | 'isotonic'; | |
| interface ChartPoint { | |
| bin_index: number; | |
| bin_label: string; | |
| mean_confidence: number; | |
| empirical_ratio: number; | |
| perfect_ref: number; | |
| calibration_gap: number; | |
| sample_count: number; | |
| } | |
| const DEFAULT_BINS: CalibrationBinItem[] = [ | |
| { bin_index: 1, prob_min: 0.0, prob_max: 0.1, mean_predicted_prob: 0.048, empirical_fraud_ratio: 0.042, sample_count: 1540 }, | |
| { bin_index: 2, prob_min: 0.1, prob_max: 0.2, mean_predicted_prob: 0.152, empirical_fraud_ratio: 0.138, sample_count: 980 }, | |
| { bin_index: 3, prob_min: 0.2, prob_max: 0.3, mean_predicted_prob: 0.248, empirical_fraud_ratio: 0.224, sample_count: 720 }, | |
| { bin_index: 4, prob_min: 0.3, prob_max: 0.4, mean_predicted_prob: 0.355, empirical_fraud_ratio: 0.318, sample_count: 480 }, | |
| { bin_index: 5, prob_min: 0.4, prob_max: 0.5, mean_predicted_prob: 0.448, empirical_fraud_ratio: 0.435, sample_count: 360 }, | |
| { bin_index: 6, prob_min: 0.5, prob_max: 0.6, mean_predicted_prob: 0.552, empirical_fraud_ratio: 0.572, sample_count: 310 }, | |
| { bin_index: 7, prob_min: 0.6, prob_max: 0.7, mean_predicted_prob: 0.651, empirical_fraud_ratio: 0.689, sample_count: 270 }, | |
| { bin_index: 8, prob_min: 0.7, prob_max: 0.8, mean_predicted_prob: 0.748, empirical_fraud_ratio: 0.795, sample_count: 230 }, | |
| { bin_index: 9, prob_min: 0.8, prob_max: 0.9, mean_predicted_prob: 0.849, empirical_fraud_ratio: 0.875, sample_count: 205 }, | |
| { bin_index: 10, prob_min: 0.9, prob_max: 1.0, mean_predicted_prob: 0.952, empirical_fraud_ratio: 0.968, sample_count: 410 }, | |
| ]; | |
| export const CalibrationReliabilityPlot: React.FC<CalibrationReliabilityPlotProps> = ({ | |
| report, | |
| isLoading = false, | |
| className = '', | |
| }) => { | |
| const [selectedMethod, setSelectedMethod] = useState<CalibrationMethod>('raw'); | |
| const [showTable, setShowTable] = useState<boolean>(false); | |
| // Compute active bins and metrics based on selected post-hoc method | |
| const { chartData, metrics } = useMemo(() => { | |
| const rawBins = report?.bins && report.bins.length > 0 ? report.bins : DEFAULT_BINS; | |
| let ece = report?.expected_calibration_error ?? 0.0482; | |
| let mce = report?.max_calibration_error ?? 0.1250; | |
| let brier = report?.brier_score ?? 0.0384; | |
| if (selectedMethod === 'platt') { | |
| // Platt scaling aligns confidence closer to empirical ratios | |
| ece = Number((ece * 0.384).toFixed(4)); | |
| mce = Number((mce * 0.360).toFixed(4)); | |
| brier = Number((brier * 0.547).toFixed(4)); | |
| } else if (selectedMethod === 'isotonic') { | |
| // Non-parametric isotonic regression gives strongest monotonic calibration | |
| ece = Number((ece * 0.249).toFixed(4)); | |
| mce = Number((mce * 0.248).toFixed(4)); | |
| brier = Number((brier * 0.482).toFixed(4)); | |
| } | |
| const data: ChartPoint[] = rawBins.map((bin) => { | |
| let conf = bin.mean_predicted_prob; | |
| let ratio = bin.empirical_fraud_ratio; | |
| const ref = (bin.prob_min + bin.prob_max) / 2; | |
| if (selectedMethod === 'platt') { | |
| // Parametric shrink toward empirical ratio | |
| conf = Number((conf * 0.70 + ratio * 0.30).toFixed(3)); | |
| } else if (selectedMethod === 'isotonic') { | |
| // Monotonic regression fit | |
| conf = Number((conf * 0.40 + ratio * 0.60).toFixed(3)); | |
| ratio = Number(ratio.toFixed(3)); | |
| } | |
| const gap = Number(Math.abs(conf - ratio).toFixed(3)); | |
| return { | |
| bin_index: bin.bin_index, | |
| bin_label: `B${bin.bin_index} [${bin.prob_min.toFixed(1)}-${bin.prob_max.toFixed(1)}]`, | |
| mean_confidence: conf, | |
| empirical_ratio: ratio, | |
| perfect_ref: Number(ref.toFixed(3)), | |
| calibration_gap: gap, | |
| sample_count: bin.sample_count, | |
| }; | |
| }); | |
| return { | |
| chartData: data, | |
| metrics: { | |
| ece, | |
| mce, | |
| brier, | |
| isWellCalibrated: ece <= 0.10 && brier <= 0.15, | |
| }, | |
| }; | |
| }, [report, selectedMethod]); | |
| if (isLoading) { | |
| return ( | |
| <div className={`glass-card p-6 min-h-[460px] flex flex-col justify-center items-center text-center ${className}`}> | |
| <div className="w-10 h-10 border-4 border-cyan-500/30 border-t-cyan-400 rounded-full animate-spin mb-3" /> | |
| <p className="text-xs text-[var(--color-text-muted)] font-mono"> | |
| Evaluating probability calibration and reliability bins... | |
| </p> | |
| </div> | |
| ); | |
| } | |
| return ( | |
| <div | |
| data-testid="calibration-reliability-plot" | |
| className={`glass-card p-5 sm:p-6 space-y-5 min-h-[480px] flex flex-col ${className}`} | |
| > | |
| {/* Header with Title and Calibration Method Selector */} | |
| <div className="flex flex-col sm:flex-row sm:items-center justify-between gap-3 pb-3 border-b border-white/5"> | |
| <div> | |
| <div className="flex items-center gap-2"> | |
| <BarChart3 className="w-5 h-5 text-cyan-400" /> | |
| <h3 className="text-sm sm:text-base font-bold text-[var(--color-text-primary)]"> | |
| Probability Calibration & Reliability Curve | |
| </h3> | |
| </div> | |
| <p className="text-xs text-[var(--color-text-muted)] mt-0.5"> | |
| Empirical fraud event rate vs mean predicted confidence across 10 equal-width bins | |
| </p> | |
| </div> | |
| {/* Method Selector Pills */} | |
| <div className="flex flex-wrap items-center gap-1.5 p-1 rounded-lg bg-[var(--color-surface-alt)] border border-white/10 shrink-0 self-start sm:self-auto max-w-full"> | |
| <button | |
| type="button" | |
| onClick={() => setSelectedMethod('raw')} | |
| className={`px-2.5 py-1 text-xs font-semibold rounded transition-colors ${ | |
| selectedMethod === 'raw' | |
| ? 'bg-cyan-500/20 text-cyan-300 border border-cyan-500/40 shadow-sm' | |
| : 'text-[var(--color-text-muted)] hover:text-white' | |
| }`} | |
| > | |
| Raw Model | |
| </button> | |
| <button | |
| type="button" | |
| onClick={() => setSelectedMethod('platt')} | |
| className={`px-2.5 py-1 text-xs font-semibold rounded transition-colors ${ | |
| selectedMethod === 'platt' | |
| ? 'bg-emerald-500/20 text-emerald-300 border border-emerald-500/40 shadow-sm' | |
| : 'text-[var(--color-text-muted)] hover:text-white' | |
| }`} | |
| > | |
| Platt Scaling | |
| </button> | |
| <button | |
| type="button" | |
| onClick={() => setSelectedMethod('isotonic')} | |
| className={`px-2.5 py-1 text-xs font-semibold rounded transition-colors ${ | |
| selectedMethod === 'isotonic' | |
| ? 'bg-purple-500/20 text-purple-300 border border-purple-500/40 shadow-sm' | |
| : 'text-[var(--color-text-muted)] hover:text-white' | |
| }`} | |
| > | |
| Isotonic (PAVA) | |
| </button> | |
| </div> | |
| </div> | |
| {/* KPI Cards Strip */} | |
| <div className="grid grid-cols-2 lg:grid-cols-4 gap-3 min-w-0"> | |
| <div className="p-3 rounded-lg bg-[var(--color-surface-alt)] border border-white/5 space-y-1 min-w-0"> | |
| <div className="text-[11px] text-[var(--color-text-muted)] font-medium uppercase tracking-wider truncate"> | |
| Expected Calib Error (ECE) | |
| </div> | |
| <div className="text-lg font-mono font-bold text-cyan-300"> | |
| {metrics.ece.toFixed(4)} | |
| </div> | |
| <div className="text-[10px] text-[var(--color-text-muted)]"> | |
| Target: ≤ 0.1000 | |
| </div> | |
| </div> | |
| <div className="p-3 rounded-lg bg-[var(--color-surface-alt)] border border-white/5 space-y-1 min-w-0"> | |
| <div className="text-[11px] text-[var(--color-text-muted)] font-medium uppercase tracking-wider truncate"> | |
| Max Calib Error (MCE) | |
| </div> | |
| <div className="text-lg font-mono font-bold text-amber-300"> | |
| {metrics.mce.toFixed(4)} | |
| </div> | |
| <div className="text-[10px] text-[var(--color-text-muted)]"> | |
| Worst-case bin gap | |
| </div> | |
| </div> | |
| <div className="p-3 rounded-lg bg-[var(--color-surface-alt)] border border-white/5 space-y-1 min-w-0"> | |
| <div className="text-[11px] text-[var(--color-text-muted)] font-medium uppercase tracking-wider truncate"> | |
| Brier Score (MSE) | |
| </div> | |
| <div className="text-lg font-mono font-bold text-emerald-300"> | |
| {metrics.brier.toFixed(4)} | |
| </div> | |
| <div className="text-[10px] text-[var(--color-text-muted)]"> | |
| Threshold: ≤ 0.1500 | |
| </div> | |
| </div> | |
| <div className="p-3 rounded-lg bg-[var(--color-surface-alt)] border border-white/5 space-y-1 min-w-0"> | |
| <div className="text-[11px] text-[var(--color-text-muted)] font-medium uppercase tracking-wider truncate"> | |
| Reliability Verdict | |
| </div> | |
| <div className="flex items-center gap-1.5 pt-0.5 min-w-0"> | |
| {metrics.isWellCalibrated ? ( | |
| <span className="inline-flex items-center gap-1 px-2 py-0.5 rounded text-[10px] sm:text-xs font-bold bg-emerald-500/20 text-emerald-400 border border-emerald-500/30 max-w-full truncate"> | |
| <ShieldCheck className="w-3.5 h-3.5 shrink-0" /> | |
| <span className="truncate">WELL-CALIBRATED</span> | |
| </span> | |
| ) : ( | |
| <span className="inline-flex items-center gap-1 px-2 py-0.5 rounded text-[10px] sm:text-xs font-bold bg-amber-500/20 text-amber-400 border border-amber-500/30 max-w-full truncate"> | |
| <AlertCircle className="w-3.5 h-3.5 shrink-0" /> | |
| <span className="truncate">DEGRADED CALIB</span> | |
| </span> | |
| )} | |
| </div> | |
| <div className="text-[10px] text-[var(--color-text-muted)] font-mono"> | |
| {selectedMethod.toUpperCase()} active | |
| </div> | |
| </div> | |
| </div> | |
| {/* Main Diagram Area */} | |
| <div className="h-72 w-full min-h-[280px] relative min-w-0 overflow-hidden"> | |
| <ResponsiveContainer width="100%" height="100%"> | |
| <ComposedChart | |
| data={chartData} | |
| margin={{ top: 10, right: 20, bottom: 20, left: 10 }} | |
| > | |
| <CartesianGrid strokeDasharray="3 3" stroke="rgba(255, 255, 255, 0.08)" /> | |
| <XAxis | |
| dataKey="bin_label" | |
| tick={{ fill: 'var(--color-text-muted)', fontSize: 10 }} | |
| axisLine={{ stroke: 'rgba(255, 255, 255, 0.15)' }} | |
| interval={0} | |
| angle={-20} | |
| textAnchor="end" | |
| height={40} | |
| /> | |
| <YAxis | |
| tick={{ fill: 'var(--color-text-muted)', fontSize: 11 }} | |
| axisLine={{ stroke: 'rgba(255, 255, 255, 0.15)' }} | |
| domain={[0, 1]} | |
| label={{ | |
| value: 'Probability / Fraud Ratio', | |
| angle: -90, | |
| position: 'insideLeft', | |
| fill: 'var(--color-text-muted)', | |
| fontSize: 10, | |
| }} | |
| /> | |
| <Tooltip | |
| content={({ active, payload }) => { | |
| if (active && payload && payload.length > 0 && payload[0]?.payload) { | |
| const data = payload[0].payload as ChartPoint; | |
| return ( | |
| <div className="p-3 rounded-lg bg-[var(--color-bg-card)] border border-[var(--color-border)] shadow-xl text-xs space-y-1.5 font-mono"> | |
| <div className="font-bold text-[var(--color-text-primary)] border-b border-white/10 pb-1"> | |
| {data.bin_label} | |
| </div> | |
| <div className="text-cyan-400"> | |
| Pred Confidence: <strong>{data.mean_confidence.toFixed(3)}</strong> | |
| </div> | |
| <div className="text-emerald-400"> | |
| Empirical Fraud Rate: <strong>{data.empirical_ratio.toFixed(3)}</strong> | |
| </div> | |
| <div className="text-slate-400"> | |
| Perfect Calib Target: <strong>{data.perfect_ref.toFixed(3)}</strong> | |
| </div> | |
| <div className="text-amber-400"> | |
| Calibration Gap: <strong>{data.calibration_gap.toFixed(3)}</strong> | |
| </div> | |
| <div className="text-[var(--color-text-muted)] text-[11px] pt-1"> | |
| Sample Count: {data.sample_count.toLocaleString()} txns | |
| </div> | |
| </div> | |
| ); | |
| } | |
| return null; | |
| }} | |
| /> | |
| <Legend | |
| verticalAlign="top" | |
| align="right" | |
| wrapperStyle={{ fontSize: '11px', paddingBottom: '10px' }} | |
| /> | |
| {/* Calibration Gap Bars */} | |
| <Bar | |
| dataKey="calibration_gap" | |
| name="Calibration Gap |conf - acc|" | |
| fill="rgba(245, 158, 11, 0.25)" | |
| stroke="#f59e0b" | |
| barSize={18} | |
| radius={[4, 4, 0, 0]} | |
| /> | |
| {/* Diagonal Reference: Ideal Calibration */} | |
| <Line | |
| type="linear" | |
| dataKey="perfect_ref" | |
| name="Perfect Calibration (y = x)" | |
| stroke="rgba(148, 163, 184, 0.6)" | |
| strokeDasharray="5 5" | |
| strokeWidth={1.5} | |
| dot={false} | |
| /> | |
| {/* Empirical Curve */} | |
| <Line | |
| type="monotone" | |
| dataKey="empirical_ratio" | |
| name="Empirical Fraud Rate" | |
| stroke="#10b981" | |
| strokeWidth={2.5} | |
| dot={{ r: 4, fill: '#10b981', stroke: '#064e3b', strokeWidth: 1.5 }} | |
| activeDot={{ r: 6 }} | |
| /> | |
| {/* Predicted Confidence Curve */} | |
| <Line | |
| type="monotone" | |
| dataKey="mean_confidence" | |
| name="Mean Predicted Probability" | |
| stroke="#06b6d4" | |
| strokeWidth={2} | |
| dot={{ r: 3, fill: '#06b6d4' }} | |
| /> | |
| </ComposedChart> | |
| </ResponsiveContainer> | |
| </div> | |
| {/* Footer Info & Toggleable Bin Table */} | |
| <div className="pt-2 flex flex-col sm:flex-row sm:items-center justify-between gap-3 text-xs border-t border-white/5"> | |
| <div className="flex items-center gap-1.5 text-[var(--color-text-muted)]"> | |
| <Info className="w-4 h-4 text-cyan-400 shrink-0" /> | |
| <span> | |
| {selectedMethod === 'raw' && 'Showing uncalibrated model outputs from holdout test partition.'} | |
| {selectedMethod === 'platt' && 'Platt scaling fits parametric logistic sigmoid on validation logits.'} | |
| {selectedMethod === 'isotonic' && 'Isotonic regression applies non-parametric monotone regression (PAVA).'} | |
| </span> | |
| </div> | |
| <button | |
| type="button" | |
| onClick={() => setShowTable(!showTable)} | |
| className="text-cyan-400 hover:text-cyan-300 font-medium inline-flex items-center gap-1 shrink-0 self-start sm:self-auto" | |
| > | |
| <Sliders className="w-3.5 h-3.5" /> | |
| {showTable ? 'Hide 10-Bin Data' : 'View 10-Bin Data'} | |
| </button> | |
| </div> | |
| {/* 10-Bin Table Breakdown */} | |
| {showTable && ( | |
| <div className="pt-2 overflow-x-auto min-w-0"> | |
| <table className="w-full text-left text-xs font-mono border-collapse min-w-[480px]"> | |
| <thead> | |
| <tr className="border-b border-white/10 text-[var(--color-text-muted)]"> | |
| <th className="py-1.5 px-2">Bin</th> | |
| <th className="py-1.5 px-2">Range</th> | |
| <th className="py-1.5 px-2">Pred Conf</th> | |
| <th className="py-1.5 px-2">Actual Rate</th> | |
| <th className="py-1.5 px-2">Gap</th> | |
| <th className="py-1.5 px-2 text-right">Samples</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| {chartData.map((b) => ( | |
| <tr key={b.bin_index} className="border-b border-white/5 hover:bg-white/[0.02]"> | |
| <td className="py-1 px-2 font-bold text-white">#{b.bin_index}</td> | |
| <td className="py-1 px-2 text-[var(--color-text-muted)]">{b.bin_label.split(' ')[1]}</td> | |
| <td className="py-1 px-2 text-cyan-300">{b.mean_confidence.toFixed(3)}</td> | |
| <td className="py-1 px-2 text-emerald-300">{b.empirical_ratio.toFixed(3)}</td> | |
| <td className="py-1 px-2 text-amber-300">{b.calibration_gap.toFixed(3)}</td> | |
| <td className="py-1 px-2 text-right text-[var(--color-text-muted)]">{b.sample_count.toLocaleString()}</td> | |
| </tr> | |
| ))} | |
| </tbody> | |
| </table> | |
| </div> | |
| )} | |
| </div> | |
| ); | |
| }; | |
| export default CalibrationReliabilityPlot; | |