/**
* Model-vs-model comparison: candidate (selected run) against the baseline
* chosen in the header. Headline deltas, per-class deltas, top regressions,
* and the PROMOTE / DO_NOT_PROMOTE recommendation with blockers.
*/
import Alert from '@mui/material/Alert';
import AlertTitle from '@mui/material/AlertTitle';
import Box from '@mui/material/Box';
import Table from '@mui/material/Table';
import TableBody from '@mui/material/TableBody';
import TableCell from '@mui/material/TableCell';
import TableHead from '@mui/material/TableHead';
import TableRow from '@mui/material/TableRow';
import Typography from '@mui/material/Typography';
import { compareRuns, fmtCompact } from '../../services/megaeval';
import { usePoll } from '../../services/labeleval';
import { ErrorNote, LoadingBox, SectionCard, fmtPct } from '../labeleval/shared';
import { ExplainTip, InfoDot } from '../help/InfoTip';
import { DeltaText } from './shared';
/** Metrics where a positive delta is a regression rather than an improvement. */
const LOWER_IS_BETTER = new Set(['anomaly_rate', 'error_rate', 'fp', 'fn', 'loc_err', 'anomalies']);
function fmtMetricValue(v: number): string {
// Rates arrive as fractions; counts as integers > 1.
return Math.abs(v) <= 1 ? fmtPct(v) : fmtCompact(v);
}
export default function CompareTab({
runId,
baselineRunId,
refreshKey,
}: {
runId: string;
baselineRunId: string | null;
refreshKey: number;
}) {
const cmp = usePoll(
() =>
baselineRunId
? compareRuns({ candidate_run_id: runId, baseline_run_id: baselineRunId })
: Promise.resolve(null),
null,
[runId, baselineRunId, refreshKey]
);
if (!baselineRunId) {
return (
Select a baseline run in the header to compare the current candidate against it.
);
}
if (cmp.error) return ;
if (cmp.loading && !cmp.data) return ;
const data = cmp.data;
if (!data) return null;
const promote = data.recommendation === 'PROMOTE';
return (
{promote ? 'PROMOTE' : 'DO NOT PROMOTE'}
— {data.candidate.model_version} vs baseline {data.baseline.model_version}
{promote
? 'Every policy rule passed: no headline or safety metric dropped beyond tolerance and no cohort regressed materially.'
: 'Each line below is a violated promotion-policy rule, with the measured drop and the maximum the policy allows.'}
{data.blockers.length ? (
{data.blockers.map((b, i) => (
{b}
))}
) : (
No blockers under the current promotion policy.
)}
Metric
Baseline
Candidate
Δ
{data.headline_deltas.map((d) => (
{d.metric}
{fmtMetricValue(d.baseline)}
{fmtMetricValue(d.candidate)}
))}
Class
n
Recall
Δ recall
Precision
Δ precision
{data.per_class.map((c) => (
{c.class}
{fmtCompact(c.n)}
{fmtPct(c.recall_baseline)} → {fmtPct(c.recall_candidate)}
{fmtPct(c.precision_baseline)} → {fmtPct(c.precision_candidate)}
))}
{`Top regressions (${data.regressions.length})`}
}
>
{data.regressions.length === 0 ? (
No cohort regressions detected against the baseline.
) : (
{data.regressions.map((r) => (
🚨 REGRESSION:
{r.cohort} recall {fmtPct(r.recall_baseline)} → {fmtPct(r.recall_candidate)} (Δ{' '}
)
n={fmtCompact(r.n)}
))}
)}
{data.worst_cohorts.length ? (
Cohort
n
Recall (baseline → candidate)
Δ
{data.worst_cohorts.map((c) => (
{c.cohort}
{fmtCompact(c.n)}
{fmtPct(c.recall_baseline)} → {fmtPct(c.recall_candidate)}
))}
) : null}
);
}