Spaces:
Paused
Paused
Deploy Sensorflow Studio (part 2)
Browse files- src/pages/hillclimb/DashboardView.tsx +187 -0
- src/pages/hillclimb/DesignLabView.tsx +248 -0
- src/pages/hillclimb/DiagnosticView.tsx +142 -0
- src/pages/hillclimb/EvidenceView.tsx +81 -0
- src/pages/hillclimb/HillClimbSection.tsx +91 -0
- src/pages/hillclimb/InterviewView.tsx +175 -0
- src/pages/hillclimb/MatrixView.tsx +151 -0
- src/pages/hillclimb/PracticeView.tsx +232 -0
- src/pages/hillclimb/SimulationView.tsx +247 -0
- src/pages/hillclimb/StarView.tsx +178 -0
- src/services/hillclimb.ts +139 -0
- src/types/hillclimb.ts +424 -0
- tests/test_hillclimb/__init__.py +0 -0
- tests/test_hillclimb/conftest.py +67 -0
- tests/test_hillclimb/test_adaptive.py +50 -0
- tests/test_hillclimb/test_api.py +142 -0
- tests/test_hillclimb/test_blueprint.py +65 -0
- tests/test_hillclimb/test_content.py +65 -0
- tests/test_hillclimb/test_design_lab.py +107 -0
- tests/test_hillclimb/test_evaluate.py +85 -0
- tests/test_hillclimb/test_interviewer.py +71 -0
- tests/test_hillclimb/test_journey.py +90 -0
- tests/test_hillclimb/test_readiness.py +83 -0
- tests/test_hillclimb/test_simulation.py +87 -0
- tests/test_hillclimb/test_star.py +64 -0
src/pages/hillclimb/DashboardView.tsx
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Dashboard: readiness bars, bottleneck callout, ONE next-best-action card, journey state. */
|
| 2 |
+
|
| 3 |
+
import Alert from '@mui/material/Alert';
|
| 4 |
+
import Box from '@mui/material/Box';
|
| 5 |
+
import Button from '@mui/material/Button';
|
| 6 |
+
import Chip from '@mui/material/Chip';
|
| 7 |
+
import Typography from '@mui/material/Typography';
|
| 8 |
+
import { ArrowRight, Compass, Mountain } from 'lucide-react';
|
| 9 |
+
import { ErrorNote, HBar, LoadingBox, SectionCard } from '../../components/labeleval/shared';
|
| 10 |
+
import { useAsync } from '../../components/hillclimb/shared';
|
| 11 |
+
import { getJourney, getNextAction, getReadiness } from '../../services/hillclimb';
|
| 12 |
+
import type { Exercise, JourneyStateId } from '../../types/hillclimb';
|
| 13 |
+
import type { GoFn } from './HillClimbSection';
|
| 14 |
+
|
| 15 |
+
const JOURNEY_ORDER: JourneyStateId[] = [
|
| 16 |
+
'NOT_STARTED', 'DIAGNOSTIC', 'LEARNING', 'PRACTICE', 'ASSESSMENT', 'REMEDIATION', 'REASSESS',
|
| 17 |
+
];
|
| 18 |
+
|
| 19 |
+
const DIM_COLORS: Record<string, string> = {
|
| 20 |
+
'Technical Depth': '#4fc3f7',
|
| 21 |
+
'System Design': '#9575cd',
|
| 22 |
+
Execution: '#ffb74d',
|
| 23 |
+
Leadership: '#66bb6a',
|
| 24 |
+
Communication: '#f06292',
|
| 25 |
+
'Safety/Risk': '#ef5350',
|
| 26 |
+
};
|
| 27 |
+
|
| 28 |
+
export default function DashboardView({ go }: { go: GoFn }) {
|
| 29 |
+
const readiness = useAsync(getReadiness);
|
| 30 |
+
const nba = useAsync(getNextAction);
|
| 31 |
+
const journey = useAsync(getJourney);
|
| 32 |
+
|
| 33 |
+
if (readiness.loading && !readiness.data) return <LoadingBox label="Computing readiness…" />;
|
| 34 |
+
if (readiness.error && !readiness.data) return <ErrorNote error={readiness.error} />;
|
| 35 |
+
|
| 36 |
+
const bn = readiness.data?.bottleneck ?? null;
|
| 37 |
+
const dims = readiness.data?.dimensions ?? [];
|
| 38 |
+
const j = journey.data;
|
| 39 |
+
|
| 40 |
+
return (
|
| 41 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 42 |
+
{j ? (
|
| 43 |
+
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.75, flexWrap: 'wrap' }}>
|
| 44 |
+
<Typography variant="caption" sx={{ color: '#8a949e', mr: 0.5 }}>
|
| 45 |
+
JOURNEY
|
| 46 |
+
</Typography>
|
| 47 |
+
{JOURNEY_ORDER.map((s) => (
|
| 48 |
+
<Chip
|
| 49 |
+
key={s}
|
| 50 |
+
size="small"
|
| 51 |
+
label={s.replace(/_/g, ' ')}
|
| 52 |
+
sx={{
|
| 53 |
+
height: 20,
|
| 54 |
+
fontSize: 10.5,
|
| 55 |
+
fontWeight: j.state === s ? 800 : 500,
|
| 56 |
+
bgcolor: j.state === s ? '#0d47a1' : '#1a2027',
|
| 57 |
+
color: j.state === s ? '#90caf9' : '#5c6873',
|
| 58 |
+
}}
|
| 59 |
+
/>
|
| 60 |
+
))}
|
| 61 |
+
{j.remediation_target ? (
|
| 62 |
+
<Chip
|
| 63 |
+
size="small"
|
| 64 |
+
label={`remediating: ${j.remediation_target}`}
|
| 65 |
+
sx={{ height: 20, fontSize: 10.5, bgcolor: '#e65100', color: '#ffe0b2' }}
|
| 66 |
+
/>
|
| 67 |
+
) : null}
|
| 68 |
+
</Box>
|
| 69 |
+
) : null}
|
| 70 |
+
|
| 71 |
+
{bn ? (
|
| 72 |
+
<Alert
|
| 73 |
+
severity="warning"
|
| 74 |
+
variant="outlined"
|
| 75 |
+
icon={<Mountain size={20} />}
|
| 76 |
+
sx={{ '& .MuiAlert-message': { width: '100%' } }}
|
| 77 |
+
>
|
| 78 |
+
<Typography variant="subtitle2" sx={{ fontWeight: 800 }}>
|
| 79 |
+
Current bottleneck: {bn.name}{' '}
|
| 80 |
+
<Chip
|
| 81 |
+
size="small"
|
| 82 |
+
label={`phase ${bn.phase} · ${bn.dimension}`}
|
| 83 |
+
sx={{ ml: 0.5, height: 18, fontSize: 10, bgcolor: '#232a31', color: '#aab4be' }}
|
| 84 |
+
/>
|
| 85 |
+
</Typography>
|
| 86 |
+
<Typography variant="body2" sx={{ color: '#cfd8dc', mt: 0.5 }}>
|
| 87 |
+
{bn.explanation}
|
| 88 |
+
</Typography>
|
| 89 |
+
</Alert>
|
| 90 |
+
) : readiness.data ? (
|
| 91 |
+
<Alert severity="success" variant="outlined">
|
| 92 |
+
No bottleneck: every competency is at COMPETENT or better. Keep polishing toward
|
| 93 |
+
INTERVIEW_READY.
|
| 94 |
+
</Alert>
|
| 95 |
+
) : null}
|
| 96 |
+
|
| 97 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap' }}>
|
| 98 |
+
<SectionCard
|
| 99 |
+
title="Readiness by dimension"
|
| 100 |
+
help="Aggregated readiness across the competency matrix, grouped into the six dimensions an EM interview loop actually probes. Bars show average score (1-5) over competencies with any signal; the fraction shows how many are COMPETENT or better."
|
| 101 |
+
sx={{ flex: '2 1 420px' }}
|
| 102 |
+
>
|
| 103 |
+
{dims.map((d) => (
|
| 104 |
+
<HBar
|
| 105 |
+
key={d.dimension}
|
| 106 |
+
label={`${d.dimension} (${d.competent}/${d.total} competent)`}
|
| 107 |
+
value={d.avg_score}
|
| 108 |
+
max={5}
|
| 109 |
+
color={DIM_COLORS[d.dimension] ?? '#4fc3f7'}
|
| 110 |
+
valueLabel={d.avg_score > 0 ? `${d.avg_score.toFixed(1)}/5` : 'no signal'}
|
| 111 |
+
/>
|
| 112 |
+
))}
|
| 113 |
+
<Typography variant="caption" sx={{ color: '#5c6873' }}>
|
| 114 |
+
Knowledge, application and evidence are tracked separately per competency — open the
|
| 115 |
+
Competency Matrix for the full explainable breakdown.
|
| 116 |
+
</Typography>
|
| 117 |
+
</SectionCard>
|
| 118 |
+
|
| 119 |
+
<SectionCard
|
| 120 |
+
title={
|
| 121 |
+
<Box component="span" sx={{ display: 'inline-flex', alignItems: 'center', gap: 0.75 }}>
|
| 122 |
+
<Compass size={16} /> Next best action
|
| 123 |
+
</Box>
|
| 124 |
+
}
|
| 125 |
+
help="Exactly one concept, one exercise, one assessment — chosen by bottleneck analysis (the prerequisite blocking the most downstream competencies, not merely your lowest score)."
|
| 126 |
+
sx={{ flex: '1 1 340px' }}
|
| 127 |
+
>
|
| 128 |
+
{nba.loading && !nba.data ? <LoadingBox label="Choosing your next move…" /> : null}
|
| 129 |
+
{nba.error && !nba.data ? <ErrorNote error={nba.error} /> : null}
|
| 130 |
+
{nba.data ? (
|
| 131 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1.25 }}>
|
| 132 |
+
<Box>
|
| 133 |
+
<Typography variant="caption" sx={{ color: '#4fc3f7', fontWeight: 700 }}>
|
| 134 |
+
1 · CONCEPT
|
| 135 |
+
</Typography>
|
| 136 |
+
<Typography variant="body2" sx={{ fontWeight: 700 }}>
|
| 137 |
+
{nba.data.concept.name}
|
| 138 |
+
</Typography>
|
| 139 |
+
<Typography variant="caption" sx={{ color: '#8a949e' }}>
|
| 140 |
+
Study: {nba.data.concept.study.join(', ') || nba.data.concept.description}
|
| 141 |
+
</Typography>
|
| 142 |
+
</Box>
|
| 143 |
+
<Box>
|
| 144 |
+
<Typography variant="caption" sx={{ color: '#9ccc65', fontWeight: 700 }}>
|
| 145 |
+
2 · EXERCISE
|
| 146 |
+
</Typography>
|
| 147 |
+
<Typography variant="body2" sx={{ color: '#cfd8dc' }}>
|
| 148 |
+
{nba.data.exercise.scenario.slice(0, 160)}…
|
| 149 |
+
</Typography>
|
| 150 |
+
</Box>
|
| 151 |
+
<Box>
|
| 152 |
+
<Typography variant="caption" sx={{ color: '#ffb74d', fontWeight: 700 }}>
|
| 153 |
+
3 · ASSESSMENT
|
| 154 |
+
</Typography>
|
| 155 |
+
<Typography variant="caption" sx={{ color: '#8a949e', display: 'block' }}>
|
| 156 |
+
{nba.data.assessment.description}
|
| 157 |
+
</Typography>
|
| 158 |
+
</Box>
|
| 159 |
+
<Button
|
| 160 |
+
variant="contained"
|
| 161 |
+
size="small"
|
| 162 |
+
endIcon={<ArrowRight size={15} />}
|
| 163 |
+
onClick={() => go('practice', nba.data!.exercise as Exercise)}
|
| 164 |
+
>
|
| 165 |
+
Start this exercise
|
| 166 |
+
</Button>
|
| 167 |
+
</Box>
|
| 168 |
+
) : null}
|
| 169 |
+
</SectionCard>
|
| 170 |
+
</Box>
|
| 171 |
+
|
| 172 |
+
{readiness.data && dims.every((d) => d.avg_score === 0) ? (
|
| 173 |
+
<Alert
|
| 174 |
+
severity="info"
|
| 175 |
+
variant="outlined"
|
| 176 |
+
action={
|
| 177 |
+
<Button size="small" onClick={() => go('diagnostic')}>
|
| 178 |
+
Run diagnostic
|
| 179 |
+
</Button>
|
| 180 |
+
}
|
| 181 |
+
>
|
| 182 |
+
No signal yet — run the adaptive diagnostic to seed your competency matrix.
|
| 183 |
+
</Alert>
|
| 184 |
+
) : null}
|
| 185 |
+
</Box>
|
| 186 |
+
);
|
| 187 |
+
}
|
src/pages/hillclimb/DesignLabView.tsx
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Design Lab: challenge brief + pragmatic add-node/connect builder + rationale fields -> graded results. */
|
| 2 |
+
|
| 3 |
+
import { useMemo, useState } from 'react';
|
| 4 |
+
import Alert from '@mui/material/Alert';
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Button from '@mui/material/Button';
|
| 7 |
+
import Chip from '@mui/material/Chip';
|
| 8 |
+
import IconButton from '@mui/material/IconButton';
|
| 9 |
+
import MenuItem from '@mui/material/MenuItem';
|
| 10 |
+
import TextField from '@mui/material/TextField';
|
| 11 |
+
import Typography from '@mui/material/Typography';
|
| 12 |
+
import { Plus, Trash2 } from 'lucide-react';
|
| 13 |
+
import { ErrorNote, HBar, LoadingBox, SectionCard } from '../../components/labeleval/shared';
|
| 14 |
+
import { PassChip, ScoreChip, useAsync } from '../../components/hillclimb/shared';
|
| 15 |
+
import { getDesignChallenges, submitDesign } from '../../services/hillclimb';
|
| 16 |
+
import type { DesignComponent, DesignEdge, DesignGrade } from '../../types/hillclimb';
|
| 17 |
+
|
| 18 |
+
const TYPE_COLORS: Record<string, string> = {
|
| 19 |
+
source: '#8d6e63', ingestion: '#4fc3f7', stream: '#29b6f6', batch: '#0288d1',
|
| 20 |
+
storage: '#9575cd', feature: '#7e57c2', training: '#ffb74d', inference: '#ffa726',
|
| 21 |
+
eval: '#66bb6a', monitoring: '#26a69a', feedback: '#ec407a',
|
| 22 |
+
};
|
| 23 |
+
|
| 24 |
+
export default function DesignLabView() {
|
| 25 |
+
const catalog = useAsync(getDesignChallenges);
|
| 26 |
+
const [challengeId, setChallengeId] = useState('petabyte_ingestion');
|
| 27 |
+
const [components, setComponents] = useState<DesignComponent[]>([]);
|
| 28 |
+
const [edges, setEdges] = useState<DesignEdge[]>([]);
|
| 29 |
+
const [rationales, setRationales] = useState<Record<string, string>>({});
|
| 30 |
+
const [newType, setNewType] = useState('source');
|
| 31 |
+
const [newName, setNewName] = useState('');
|
| 32 |
+
const [newNote, setNewNote] = useState('');
|
| 33 |
+
const [edgeFrom, setEdgeFrom] = useState('');
|
| 34 |
+
const [edgeTo, setEdgeTo] = useState('');
|
| 35 |
+
const [grade, setGrade] = useState<DesignGrade | null>(null);
|
| 36 |
+
const [busy, setBusy] = useState(false);
|
| 37 |
+
const [error, setError] = useState<string | null>(null);
|
| 38 |
+
|
| 39 |
+
const challenge = useMemo(
|
| 40 |
+
() => catalog.data?.challenges.find((c) => c.challenge_id === challengeId) ?? null,
|
| 41 |
+
[catalog.data, challengeId]
|
| 42 |
+
);
|
| 43 |
+
|
| 44 |
+
const reset = (cid: string) => {
|
| 45 |
+
setChallengeId(cid);
|
| 46 |
+
setComponents([]);
|
| 47 |
+
setEdges([]);
|
| 48 |
+
setRationales({});
|
| 49 |
+
setGrade(null);
|
| 50 |
+
};
|
| 51 |
+
|
| 52 |
+
const addComponent = () => {
|
| 53 |
+
const id = `c${components.length + 1}`;
|
| 54 |
+
setComponents([...components, { id, type: newType, name: newName || `${newType}-${id}`, note: newNote }]);
|
| 55 |
+
setNewName('');
|
| 56 |
+
setNewNote('');
|
| 57 |
+
};
|
| 58 |
+
|
| 59 |
+
const removeComponent = (id: string) => {
|
| 60 |
+
setComponents(components.filter((c) => c.id !== id));
|
| 61 |
+
setEdges(edges.filter((e) => e.source !== id && e.target !== id));
|
| 62 |
+
};
|
| 63 |
+
|
| 64 |
+
const addEdge = () => {
|
| 65 |
+
if (!edgeFrom || !edgeTo || edgeFrom === edgeTo) return;
|
| 66 |
+
if (edges.some((e) => e.source === edgeFrom && e.target === edgeTo)) return;
|
| 67 |
+
setEdges([...edges, { source: edgeFrom, target: edgeTo }]);
|
| 68 |
+
};
|
| 69 |
+
|
| 70 |
+
const submit = () => {
|
| 71 |
+
setBusy(true);
|
| 72 |
+
setError(null);
|
| 73 |
+
submitDesign(challengeId, components, edges, rationales)
|
| 74 |
+
.then(setGrade)
|
| 75 |
+
.catch((e: Error) => setError(e.message))
|
| 76 |
+
.finally(() => setBusy(false));
|
| 77 |
+
};
|
| 78 |
+
|
| 79 |
+
if (catalog.loading && !catalog.data) return <LoadingBox label="Loading challenges…" />;
|
| 80 |
+
if (catalog.error && !catalog.data) return <ErrorNote error={catalog.error} />;
|
| 81 |
+
|
| 82 |
+
const nameOf = (id: string) => components.find((c) => c.id === id)?.name ?? id;
|
| 83 |
+
|
| 84 |
+
return (
|
| 85 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 86 |
+
<SectionCard
|
| 87 |
+
title="Challenge"
|
| 88 |
+
action={
|
| 89 |
+
<TextField select size="small" value={challengeId} onChange={(e) => reset(e.target.value)} sx={{ minWidth: 280 }}>
|
| 90 |
+
{(catalog.data?.challenges ?? []).map((c) => (
|
| 91 |
+
<MenuItem key={c.challenge_id} value={c.challenge_id}>
|
| 92 |
+
{c.title}
|
| 93 |
+
</MenuItem>
|
| 94 |
+
))}
|
| 95 |
+
</TextField>
|
| 96 |
+
}
|
| 97 |
+
help="Submit a typed component graph plus a written rationale per key decision. The rule-based grader checks required stages, orphans, feedback-loop closure, single points of failure, capacity math, and rationale coverage of scalability / reliability / latency / cost / observability / failure handling / tradeoffs."
|
| 98 |
+
>
|
| 99 |
+
{challenge ? (
|
| 100 |
+
<>
|
| 101 |
+
<Typography variant="body2" sx={{ color: '#e6e9ec', lineHeight: 1.65, mb: 1 }}>
|
| 102 |
+
{challenge.brief}
|
| 103 |
+
</Typography>
|
| 104 |
+
<Typography variant="caption" sx={{ color: '#8a949e' }}>
|
| 105 |
+
Required stages:{' '}
|
| 106 |
+
{challenge.required_stages.map((g) => g.join(' or ')).join(' · ')}
|
| 107 |
+
{challenge.requires_feedback_loop ? ' · must close a feedback loop' : ''}
|
| 108 |
+
</Typography>
|
| 109 |
+
</>
|
| 110 |
+
) : null}
|
| 111 |
+
</SectionCard>
|
| 112 |
+
|
| 113 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap', alignItems: 'flex-start' }}>
|
| 114 |
+
<SectionCard title={`Components (${components.length})`} sx={{ flex: '1 1 340px' }}>
|
| 115 |
+
<Box sx={{ display: 'flex', gap: 1, mb: 1.5, flexWrap: 'wrap' }}>
|
| 116 |
+
<TextField select size="small" label="Type" value={newType} onChange={(e) => setNewType(e.target.value)} sx={{ width: 130 }}>
|
| 117 |
+
{(catalog.data?.component_types ?? []).map((t) => (
|
| 118 |
+
<MenuItem key={t} value={t}>{t}</MenuItem>
|
| 119 |
+
))}
|
| 120 |
+
</TextField>
|
| 121 |
+
<TextField size="small" label="Name" value={newName} onChange={(e) => setNewName(e.target.value)} sx={{ width: 150 }} />
|
| 122 |
+
<TextField size="small" label="Capacity note (e.g. 2 PB/week)" value={newNote} onChange={(e) => setNewNote(e.target.value)} sx={{ flex: 1, minWidth: 170 }} />
|
| 123 |
+
<Button size="small" variant="outlined" startIcon={<Plus size={14} />} onClick={addComponent}>
|
| 124 |
+
Add
|
| 125 |
+
</Button>
|
| 126 |
+
</Box>
|
| 127 |
+
{components.map((c) => (
|
| 128 |
+
<Box key={c.id} sx={{ display: 'flex', alignItems: 'center', gap: 1, mb: 0.5 }}>
|
| 129 |
+
<Chip size="small" label={c.type} sx={{ bgcolor: TYPE_COLORS[c.type] ?? '#37474f', color: '#101418', fontWeight: 700, fontSize: 10.5, width: 88 }} />
|
| 130 |
+
<Typography variant="body2" sx={{ flex: 1 }}>
|
| 131 |
+
{c.name}
|
| 132 |
+
{c.note ? (
|
| 133 |
+
<Typography component="span" variant="caption" sx={{ color: '#8a949e', ml: 1 }}>
|
| 134 |
+
{c.note}
|
| 135 |
+
</Typography>
|
| 136 |
+
) : null}
|
| 137 |
+
</Typography>
|
| 138 |
+
<IconButton size="small" onClick={() => removeComponent(c.id)}>
|
| 139 |
+
<Trash2 size={14} />
|
| 140 |
+
</IconButton>
|
| 141 |
+
</Box>
|
| 142 |
+
))}
|
| 143 |
+
</SectionCard>
|
| 144 |
+
|
| 145 |
+
<SectionCard title={`Connections (${edges.length})`} sx={{ flex: '1 1 300px' }}>
|
| 146 |
+
<Box sx={{ display: 'flex', gap: 1, mb: 1.5 }}>
|
| 147 |
+
<TextField select size="small" label="From" value={edgeFrom} onChange={(e) => setEdgeFrom(e.target.value)} sx={{ flex: 1 }}>
|
| 148 |
+
{components.map((c) => (
|
| 149 |
+
<MenuItem key={c.id} value={c.id}>{c.name}</MenuItem>
|
| 150 |
+
))}
|
| 151 |
+
</TextField>
|
| 152 |
+
<TextField select size="small" label="To" value={edgeTo} onChange={(e) => setEdgeTo(e.target.value)} sx={{ flex: 1 }}>
|
| 153 |
+
{components.map((c) => (
|
| 154 |
+
<MenuItem key={c.id} value={c.id}>{c.name}</MenuItem>
|
| 155 |
+
))}
|
| 156 |
+
</TextField>
|
| 157 |
+
<Button size="small" variant="outlined" onClick={addEdge}>
|
| 158 |
+
Connect
|
| 159 |
+
</Button>
|
| 160 |
+
</Box>
|
| 161 |
+
{edges.map((e, i) => (
|
| 162 |
+
<Box key={i} sx={{ display: 'flex', alignItems: 'center', gap: 0.5, mb: 0.4 }}>
|
| 163 |
+
<Typography variant="caption" sx={{ fontFamily: 'monospace', color: '#aab4be', flex: 1 }}>
|
| 164 |
+
{nameOf(e.source)} → {nameOf(e.target)}
|
| 165 |
+
</Typography>
|
| 166 |
+
<IconButton size="small" onClick={() => setEdges(edges.filter((_x, j) => j !== i))}>
|
| 167 |
+
<Trash2 size={13} />
|
| 168 |
+
</IconButton>
|
| 169 |
+
</Box>
|
| 170 |
+
))}
|
| 171 |
+
</SectionCard>
|
| 172 |
+
</Box>
|
| 173 |
+
|
| 174 |
+
<SectionCard title="Rationale per key decision" help="Each decision needs written reasoning. The grader scans for capacity math and coverage of the seven design dimensions — hand-waving scores zero.">
|
| 175 |
+
{(challenge?.key_decisions ?? []).map((d) => (
|
| 176 |
+
<TextField
|
| 177 |
+
key={d}
|
| 178 |
+
label={d}
|
| 179 |
+
multiline
|
| 180 |
+
minRows={2}
|
| 181 |
+
fullWidth
|
| 182 |
+
sx={{ mb: 1.5 }}
|
| 183 |
+
value={rationales[d] ?? ''}
|
| 184 |
+
onChange={(e) => setRationales({ ...rationales, [d]: e.target.value })}
|
| 185 |
+
/>
|
| 186 |
+
))}
|
| 187 |
+
<Box sx={{ display: 'flex', justifyContent: 'flex-end' }}>
|
| 188 |
+
<Button variant="contained" onClick={submit} disabled={busy || components.length === 0}>
|
| 189 |
+
{busy ? 'Grading…' : 'Submit for grading'}
|
| 190 |
+
</Button>
|
| 191 |
+
</Box>
|
| 192 |
+
{error ? <ErrorNote error={error} /> : null}
|
| 193 |
+
</SectionCard>
|
| 194 |
+
|
| 195 |
+
{grade ? (
|
| 196 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap' }}>
|
| 197 |
+
<SectionCard
|
| 198 |
+
title={
|
| 199 |
+
<Box component="span" sx={{ display: 'inline-flex', alignItems: 'center', gap: 1 }}>
|
| 200 |
+
Structural checks <ScoreChip score={grade.overall_score} label="Overall" />
|
| 201 |
+
</Box>
|
| 202 |
+
}
|
| 203 |
+
sx={{ flex: '1 1 320px' }}
|
| 204 |
+
>
|
| 205 |
+
<Box sx={{ display: 'flex', gap: 0.75, flexWrap: 'wrap', mb: 1.5 }}>
|
| 206 |
+
<PassChip passed={grade.structural.missing_stages.length === 0} label="required stages" />
|
| 207 |
+
<PassChip passed={grade.structural.orphan_components.length === 0} label="no orphans" />
|
| 208 |
+
<PassChip
|
| 209 |
+
passed={!grade.structural.feedback_loop_required || grade.structural.feedback_loop_closed}
|
| 210 |
+
label="feedback loop"
|
| 211 |
+
/>
|
| 212 |
+
<PassChip passed={grade.structural.single_points_of_failure.length === 0} label="no SPOF" />
|
| 213 |
+
<PassChip passed={grade.structural.capacity_math_found} label="capacity math" />
|
| 214 |
+
</Box>
|
| 215 |
+
{grade.gaps.map((g, i) => (
|
| 216 |
+
<Typography key={i} variant="caption" sx={{ color: '#ef9a9a', display: 'block', mb: 0.5 }}>
|
| 217 |
+
• {g}
|
| 218 |
+
</Typography>
|
| 219 |
+
))}
|
| 220 |
+
{grade.gaps.length === 0 ? (
|
| 221 |
+
<Typography variant="body2" sx={{ color: '#9ccc65' }}>
|
| 222 |
+
No structural gaps found.
|
| 223 |
+
</Typography>
|
| 224 |
+
) : null}
|
| 225 |
+
</SectionCard>
|
| 226 |
+
|
| 227 |
+
<SectionCard title="Per-dimension scores" sx={{ flex: '1 1 340px' }}>
|
| 228 |
+
{grade.dimension_grades.map((d) => (
|
| 229 |
+
<HBar
|
| 230 |
+
key={d.dimension}
|
| 231 |
+
label={d.dimension.replace(/_/g, ' ')}
|
| 232 |
+
value={d.score}
|
| 233 |
+
max={5}
|
| 234 |
+
color={d.score >= 3 ? '#66bb6a' : d.score >= 2 ? '#f9a825' : '#ef5350'}
|
| 235 |
+
valueLabel={`${d.score}/5`}
|
| 236 |
+
/>
|
| 237 |
+
))}
|
| 238 |
+
{grade.evidence_id ? (
|
| 239 |
+
<Alert severity="success" variant="outlined" sx={{ mt: 1, py: 0.25 }}>
|
| 240 |
+
Saved to your evidence library.
|
| 241 |
+
</Alert>
|
| 242 |
+
) : null}
|
| 243 |
+
</SectionCard>
|
| 244 |
+
</Box>
|
| 245 |
+
) : null}
|
| 246 |
+
</Box>
|
| 247 |
+
);
|
| 248 |
+
}
|
src/pages/hillclimb/DiagnosticView.tsx
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Adaptive diagnostic: a short question set that seeds the competency matrix. */
|
| 2 |
+
|
| 3 |
+
import { useState } from 'react';
|
| 4 |
+
import Alert from '@mui/material/Alert';
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Button from '@mui/material/Button';
|
| 7 |
+
import Chip from '@mui/material/Chip';
|
| 8 |
+
import LinearProgress from '@mui/material/LinearProgress';
|
| 9 |
+
import TextField from '@mui/material/TextField';
|
| 10 |
+
import Typography from '@mui/material/Typography';
|
| 11 |
+
import { ErrorNote, SectionCard } from '../../components/labeleval/shared';
|
| 12 |
+
import { ScoreChip } from '../../components/hillclimb/shared';
|
| 13 |
+
import { answerDiagnostic, startDiagnostic } from '../../services/hillclimb';
|
| 14 |
+
import type { DiagnosticSession } from '../../types/hillclimb';
|
| 15 |
+
import type { GoFn } from './HillClimbSection';
|
| 16 |
+
|
| 17 |
+
export default function DiagnosticView({ go }: { go: GoFn }) {
|
| 18 |
+
const [session, setSession] = useState<DiagnosticSession | null>(null);
|
| 19 |
+
const [answer, setAnswer] = useState('');
|
| 20 |
+
const [busy, setBusy] = useState(false);
|
| 21 |
+
const [error, setError] = useState<string | null>(null);
|
| 22 |
+
|
| 23 |
+
const start = () => {
|
| 24 |
+
setBusy(true);
|
| 25 |
+
setError(null);
|
| 26 |
+
startDiagnostic(Math.floor(Math.random() * 100000))
|
| 27 |
+
.then(setSession)
|
| 28 |
+
.catch((e: Error) => setError(e.message))
|
| 29 |
+
.finally(() => setBusy(false));
|
| 30 |
+
};
|
| 31 |
+
|
| 32 |
+
const submit = () => {
|
| 33 |
+
if (!session || !answer.trim()) return;
|
| 34 |
+
setBusy(true);
|
| 35 |
+
setError(null);
|
| 36 |
+
answerDiagnostic(session.diagnostic_id, answer)
|
| 37 |
+
.then((s) => {
|
| 38 |
+
setSession(s);
|
| 39 |
+
setAnswer('');
|
| 40 |
+
})
|
| 41 |
+
.catch((e: Error) => setError(e.message))
|
| 42 |
+
.finally(() => setBusy(false));
|
| 43 |
+
};
|
| 44 |
+
|
| 45 |
+
return (
|
| 46 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 900 }}>
|
| 47 |
+
{error ? <ErrorNote error={error} /> : null}
|
| 48 |
+
|
| 49 |
+
{!session ? (
|
| 50 |
+
<SectionCard
|
| 51 |
+
title="Adaptive diagnostic"
|
| 52 |
+
help="Six short scenario questions. The sequence adapts: a weak answer drills into prerequisites, a strong answer jumps ahead. Results seed your competency matrix and set the journey to LEARNING."
|
| 53 |
+
>
|
| 54 |
+
<Typography variant="body2" sx={{ color: '#aab4be', mb: 2 }}>
|
| 55 |
+
Answer six scenario questions in your own words. There is no multiple choice and no
|
| 56 |
+
memorizable answer key — each question is generated from a parameterized template, and
|
| 57 |
+
your answers are scored against a concept rubric with quoted evidence.
|
| 58 |
+
</Typography>
|
| 59 |
+
<Button variant="contained" onClick={start} disabled={busy}>
|
| 60 |
+
Start diagnostic
|
| 61 |
+
</Button>
|
| 62 |
+
</SectionCard>
|
| 63 |
+
) : null}
|
| 64 |
+
|
| 65 |
+
{session && session.status === 'active' && session.current_question ? (
|
| 66 |
+
<SectionCard
|
| 67 |
+
title={`Question ${session.answered + 1} of ${session.total_questions}`}
|
| 68 |
+
action={
|
| 69 |
+
<Chip
|
| 70 |
+
size="small"
|
| 71 |
+
label={session.current_question.competency_id}
|
| 72 |
+
sx={{ bgcolor: '#232a31', fontFamily: 'monospace', fontSize: 11 }}
|
| 73 |
+
/>
|
| 74 |
+
}
|
| 75 |
+
>
|
| 76 |
+
<LinearProgress
|
| 77 |
+
variant="determinate"
|
| 78 |
+
value={(session.answered / session.total_questions) * 100}
|
| 79 |
+
sx={{ mb: 2, height: 5, borderRadius: 2 }}
|
| 80 |
+
/>
|
| 81 |
+
<Typography variant="body2" sx={{ color: '#e6e9ec', mb: 2, lineHeight: 1.65 }}>
|
| 82 |
+
{session.current_question.scenario}
|
| 83 |
+
</Typography>
|
| 84 |
+
<TextField
|
| 85 |
+
multiline
|
| 86 |
+
minRows={5}
|
| 87 |
+
fullWidth
|
| 88 |
+
placeholder="Answer in your own words — mechanisms, numbers, tradeoffs…"
|
| 89 |
+
value={answer}
|
| 90 |
+
onChange={(e) => setAnswer(e.target.value)}
|
| 91 |
+
disabled={busy}
|
| 92 |
+
/>
|
| 93 |
+
<Box sx={{ display: 'flex', justifyContent: 'flex-end', mt: 1.5, gap: 1 }}>
|
| 94 |
+
<Button variant="contained" onClick={submit} disabled={busy || !answer.trim()}>
|
| 95 |
+
{busy ? 'Scoring…' : 'Submit answer'}
|
| 96 |
+
</Button>
|
| 97 |
+
</Box>
|
| 98 |
+
{session.results.length > 0 ? (
|
| 99 |
+
<Box sx={{ display: 'flex', gap: 0.75, mt: 2, flexWrap: 'wrap' }}>
|
| 100 |
+
{session.results.map((r, i) => (
|
| 101 |
+
<Chip
|
| 102 |
+
key={i}
|
| 103 |
+
size="small"
|
| 104 |
+
label={`${r.competency_id}: ${r.score}/5`}
|
| 105 |
+
sx={{ bgcolor: '#1a2027', color: '#8a949e', fontSize: 10.5, height: 20 }}
|
| 106 |
+
/>
|
| 107 |
+
))}
|
| 108 |
+
</Box>
|
| 109 |
+
) : null}
|
| 110 |
+
</SectionCard>
|
| 111 |
+
) : null}
|
| 112 |
+
|
| 113 |
+
{session && session.status === 'complete' ? (
|
| 114 |
+
<SectionCard title="Diagnostic complete — matrix seeded">
|
| 115 |
+
<Alert severity="success" variant="outlined" sx={{ mb: 2 }}>
|
| 116 |
+
Your competency matrix has been seeded from {session.results.length} scored answers and
|
| 117 |
+
your journey advanced to LEARNING.
|
| 118 |
+
</Alert>
|
| 119 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1, mb: 2 }}>
|
| 120 |
+
{session.results.map((r, i) => (
|
| 121 |
+
<Box key={i} sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
|
| 122 |
+
<ScoreChip score={r.score} label="" />
|
| 123 |
+
<Typography variant="body2" sx={{ fontFamily: 'monospace', fontSize: 12 }}>
|
| 124 |
+
{r.competency_id}
|
| 125 |
+
</Typography>
|
| 126 |
+
<Typography variant="caption" sx={{ color: '#8a949e' }}>
|
| 127 |
+
{r.evaluation.weaknesses[0] ?? r.evaluation.strengths[0] ?? ''}
|
| 128 |
+
</Typography>
|
| 129 |
+
</Box>
|
| 130 |
+
))}
|
| 131 |
+
</Box>
|
| 132 |
+
<Box sx={{ display: 'flex', gap: 1 }}>
|
| 133 |
+
<Button variant="contained" onClick={() => go('dashboard')}>
|
| 134 |
+
See dashboard & next best action
|
| 135 |
+
</Button>
|
| 136 |
+
<Button onClick={start}>Retake (new questions)</Button>
|
| 137 |
+
</Box>
|
| 138 |
+
</SectionCard>
|
| 139 |
+
) : null}
|
| 140 |
+
</Box>
|
| 141 |
+
);
|
| 142 |
+
}
|
src/pages/hillclimb/EvidenceView.tsx
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Evidence library: every artifact backing the readiness matrix. */
|
| 2 |
+
|
| 3 |
+
import { useState } from 'react';
|
| 4 |
+
import Box from '@mui/material/Box';
|
| 5 |
+
import Chip from '@mui/material/Chip';
|
| 6 |
+
import ToggleButton from '@mui/material/ToggleButton';
|
| 7 |
+
import ToggleButtonGroup from '@mui/material/ToggleButtonGroup';
|
| 8 |
+
import Typography from '@mui/material/Typography';
|
| 9 |
+
import { EmptyState, ErrorNote, LoadingBox, SectionCard } from '../../components/labeleval/shared';
|
| 10 |
+
import { ScoreChip, useAsync } from '../../components/hillclimb/shared';
|
| 11 |
+
import { getEvidence } from '../../services/hillclimb';
|
| 12 |
+
|
| 13 |
+
const TYPE_LABELS: Record<string, string> = {
|
| 14 |
+
exercise_attempt: 'Exercise',
|
| 15 |
+
star_story: 'STAR story',
|
| 16 |
+
design_submission: 'Design lab',
|
| 17 |
+
simulation_debrief: 'Simulation',
|
| 18 |
+
interview_transcript: 'Interview',
|
| 19 |
+
diagnostic: 'Diagnostic',
|
| 20 |
+
};
|
| 21 |
+
|
| 22 |
+
export default function EvidenceView({ competencyFilter }: { competencyFilter: string | null }) {
|
| 23 |
+
const [typeFilter, setTypeFilter] = useState<string>('all');
|
| 24 |
+
const evidence = useAsync(() => getEvidence(competencyFilter ?? undefined), [competencyFilter]);
|
| 25 |
+
|
| 26 |
+
if (evidence.loading && !evidence.data) return <LoadingBox label="Loading evidence…" />;
|
| 27 |
+
if (evidence.error && !evidence.data) return <ErrorNote error={evidence.error} />;
|
| 28 |
+
|
| 29 |
+
const items = (evidence.data?.evidence ?? []).filter(
|
| 30 |
+
(e) => typeFilter === 'all' || e.artifact_type === typeFilter
|
| 31 |
+
);
|
| 32 |
+
|
| 33 |
+
return (
|
| 34 |
+
<SectionCard
|
| 35 |
+
title={`Evidence library (${items.length})`}
|
| 36 |
+
action={
|
| 37 |
+
<ToggleButtonGroup size="small" exclusive value={typeFilter} onChange={(_e, v: string | null) => v && setTypeFilter(v)}>
|
| 38 |
+
<ToggleButton value="all">All</ToggleButton>
|
| 39 |
+
{Object.entries(TYPE_LABELS).map(([k, v]) => (
|
| 40 |
+
<ToggleButton key={k} value={k}>
|
| 41 |
+
{v}
|
| 42 |
+
</ToggleButton>
|
| 43 |
+
))}
|
| 44 |
+
</ToggleButtonGroup>
|
| 45 |
+
}
|
| 46 |
+
help="Every score in the platform traces back to artifacts here: exercise attempts (with quoted answer evidence), diagnosed STAR stories, graded designs, simulation debriefs, and interview transcripts."
|
| 47 |
+
>
|
| 48 |
+
{items.length === 0 ? (
|
| 49 |
+
<EmptyState
|
| 50 |
+
title="No evidence yet"
|
| 51 |
+
message="Evidence accumulates as you practice: run the diagnostic, submit exercises, diagnose STAR stories, grade designs, finish simulations and interviews."
|
| 52 |
+
/>
|
| 53 |
+
) : (
|
| 54 |
+
items.map((e) => (
|
| 55 |
+
<Box key={e.evidence_id} sx={{ py: 1.25, borderBottom: '1px solid #232a31' }}>
|
| 56 |
+
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1, flexWrap: 'wrap' }}>
|
| 57 |
+
<Chip size="small" label={TYPE_LABELS[e.artifact_type] ?? e.artifact_type} sx={{ bgcolor: '#232a31', color: '#aab4be', fontSize: 10.5, height: 20 }} />
|
| 58 |
+
{e.score > 0 ? <ScoreChip score={Math.round(e.score)} label="" /> : null}
|
| 59 |
+
<Typography variant="caption" sx={{ color: '#5c6873' }}>
|
| 60 |
+
{new Date(e.timestamp).toLocaleString()} · {e.source}
|
| 61 |
+
</Typography>
|
| 62 |
+
</Box>
|
| 63 |
+
<Typography variant="body2" sx={{ color: '#e6e9ec', mt: 0.5 }}>
|
| 64 |
+
{e.summary}
|
| 65 |
+
</Typography>
|
| 66 |
+
{e.quotes.slice(0, 2).map((q, i) => (
|
| 67 |
+
<Typography key={i} variant="caption" sx={{ display: 'block', fontStyle: 'italic', color: '#8a949e' }}>
|
| 68 |
+
“{q}”
|
| 69 |
+
</Typography>
|
| 70 |
+
))}
|
| 71 |
+
<Box sx={{ display: 'flex', gap: 0.5, mt: 0.5, flexWrap: 'wrap' }}>
|
| 72 |
+
{e.competency_ids.map((c) => (
|
| 73 |
+
<Chip key={c} size="small" label={c} sx={{ bgcolor: '#1a2027', color: '#5c6873', fontFamily: 'monospace', fontSize: 10, height: 18 }} />
|
| 74 |
+
))}
|
| 75 |
+
</Box>
|
| 76 |
+
</Box>
|
| 77 |
+
))
|
| 78 |
+
)}
|
| 79 |
+
</SectionCard>
|
| 80 |
+
);
|
| 81 |
+
}
|
src/pages/hillclimb/HillClimbSection.tsx
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Hill Climbing EM — section shell with competency-first internal navigation:
|
| 3 |
+
* Assess → Diagnose → Practice → Apply → Evaluate → Improve (not an LMS).
|
| 4 |
+
*/
|
| 5 |
+
|
| 6 |
+
import { useCallback, useState } from 'react';
|
| 7 |
+
import Box from '@mui/material/Box';
|
| 8 |
+
import Tab from '@mui/material/Tab';
|
| 9 |
+
import Tabs from '@mui/material/Tabs';
|
| 10 |
+
import type { Exercise } from '../../types/hillclimb';
|
| 11 |
+
import DashboardView from './DashboardView';
|
| 12 |
+
import DiagnosticView from './DiagnosticView';
|
| 13 |
+
import PracticeView from './PracticeView';
|
| 14 |
+
import StarView from './StarView';
|
| 15 |
+
import DesignLabView from './DesignLabView';
|
| 16 |
+
import SimulationView from './SimulationView';
|
| 17 |
+
import InterviewView from './InterviewView';
|
| 18 |
+
import EvidenceView from './EvidenceView';
|
| 19 |
+
import MatrixView from './MatrixView';
|
| 20 |
+
|
| 21 |
+
export type HillClimbView =
|
| 22 |
+
| 'dashboard'
|
| 23 |
+
| 'diagnostic'
|
| 24 |
+
| 'practice'
|
| 25 |
+
| 'star'
|
| 26 |
+
| 'design'
|
| 27 |
+
| 'simulation'
|
| 28 |
+
| 'interview'
|
| 29 |
+
| 'matrix'
|
| 30 |
+
| 'evidence';
|
| 31 |
+
|
| 32 |
+
const VIEWS: { id: HillClimbView; label: string }[] = [
|
| 33 |
+
{ id: 'dashboard', label: 'Dashboard' },
|
| 34 |
+
{ id: 'diagnostic', label: 'Diagnostic' },
|
| 35 |
+
{ id: 'practice', label: 'Practice' },
|
| 36 |
+
{ id: 'star', label: 'STAR Story Box' },
|
| 37 |
+
{ id: 'design', label: 'Design Lab' },
|
| 38 |
+
{ id: 'simulation', label: 'Simulation' },
|
| 39 |
+
{ id: 'interview', label: 'Interview Room' },
|
| 40 |
+
{ id: 'matrix', label: 'Competency Matrix' },
|
| 41 |
+
{ id: 'evidence', label: 'Evidence Library' },
|
| 42 |
+
];
|
| 43 |
+
|
| 44 |
+
export interface GoFn {
|
| 45 |
+
(view: HillClimbView, exercise?: Exercise | null, competencyId?: string | null): void;
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
export default function HillClimbSection({ initialView }: { initialView?: string | null }) {
|
| 49 |
+
const initial = (VIEWS.some((v) => v.id === initialView) ? initialView : 'dashboard') as HillClimbView;
|
| 50 |
+
const [view, setView] = useState<HillClimbView>(initial);
|
| 51 |
+
const [seedExercise, setSeedExercise] = useState<Exercise | null>(null);
|
| 52 |
+
const [seedCompetency, setSeedCompetency] = useState<string | null>(null);
|
| 53 |
+
|
| 54 |
+
const go: GoFn = useCallback((next, exercise = null, competencyId = null) => {
|
| 55 |
+
setSeedExercise(exercise);
|
| 56 |
+
setSeedCompetency(competencyId);
|
| 57 |
+
setView(next);
|
| 58 |
+
}, []);
|
| 59 |
+
|
| 60 |
+
return (
|
| 61 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 62 |
+
<Tabs
|
| 63 |
+
value={view}
|
| 64 |
+
onChange={(_e, v: HillClimbView) => go(v)}
|
| 65 |
+
variant="scrollable"
|
| 66 |
+
scrollButtons="auto"
|
| 67 |
+
sx={{
|
| 68 |
+
minHeight: 38,
|
| 69 |
+
borderBottom: '1px solid #232a31',
|
| 70 |
+
'& .MuiTab-root': { minHeight: 38, py: 0.5, fontSize: 13, textTransform: 'none' },
|
| 71 |
+
}}
|
| 72 |
+
>
|
| 73 |
+
{VIEWS.map((v) => (
|
| 74 |
+
<Tab key={v.id} value={v.id} label={v.label} />
|
| 75 |
+
))}
|
| 76 |
+
</Tabs>
|
| 77 |
+
|
| 78 |
+
{view === 'dashboard' ? <DashboardView go={go} /> : null}
|
| 79 |
+
{view === 'diagnostic' ? <DiagnosticView go={go} /> : null}
|
| 80 |
+
{view === 'practice' ? (
|
| 81 |
+
<PracticeView go={go} seedExercise={seedExercise} seedCompetency={seedCompetency} />
|
| 82 |
+
) : null}
|
| 83 |
+
{view === 'star' ? <StarView /> : null}
|
| 84 |
+
{view === 'design' ? <DesignLabView /> : null}
|
| 85 |
+
{view === 'simulation' ? <SimulationView /> : null}
|
| 86 |
+
{view === 'interview' ? <InterviewView /> : null}
|
| 87 |
+
{view === 'matrix' ? <MatrixView go={go} /> : null}
|
| 88 |
+
{view === 'evidence' ? <EvidenceView competencyFilter={seedCompetency} /> : null}
|
| 89 |
+
</Box>
|
| 90 |
+
);
|
| 91 |
+
}
|
src/pages/hillclimb/InterviewView.tsx
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Interview room: chat-style adaptive session with per-answer evaluation chips. */
|
| 2 |
+
|
| 3 |
+
import { useState } from 'react';
|
| 4 |
+
import Alert from '@mui/material/Alert';
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Button from '@mui/material/Button';
|
| 7 |
+
import Chip from '@mui/material/Chip';
|
| 8 |
+
import TextField from '@mui/material/TextField';
|
| 9 |
+
import ToggleButton from '@mui/material/ToggleButton';
|
| 10 |
+
import ToggleButtonGroup from '@mui/material/ToggleButtonGroup';
|
| 11 |
+
import Typography from '@mui/material/Typography';
|
| 12 |
+
import { ErrorNote, SectionCard } from '../../components/labeleval/shared';
|
| 13 |
+
import { ScoreChip, scoreColor } from '../../components/hillclimb/shared';
|
| 14 |
+
import { answerInterview, endInterview, startInterview } from '../../services/hillclimb';
|
| 15 |
+
import type { InterviewSession, InterviewTurn } from '../../types/hillclimb';
|
| 16 |
+
|
| 17 |
+
const TYPE_LABEL: Record<InterviewTurn['question_type'], { label: string; color: string }> = {
|
| 18 |
+
opening: { label: 'opening', color: '#4fc3f7' },
|
| 19 |
+
probe: { label: 'weakness probe', color: '#ef5350' },
|
| 20 |
+
depth_probe: { label: 'depth probe', color: '#ffb74d' },
|
| 21 |
+
escalate: { label: 'escalation', color: '#66bb6a' },
|
| 22 |
+
advance: { label: 'new competency', color: '#9575cd' },
|
| 23 |
+
};
|
| 24 |
+
|
| 25 |
+
export default function InterviewView() {
|
| 26 |
+
const [mode, setMode] = useState('hybrid');
|
| 27 |
+
const [session, setSession] = useState<InterviewSession | null>(null);
|
| 28 |
+
const [answer, setAnswer] = useState('');
|
| 29 |
+
const [busy, setBusy] = useState(false);
|
| 30 |
+
const [error, setError] = useState<string | null>(null);
|
| 31 |
+
|
| 32 |
+
const start = () => {
|
| 33 |
+
setBusy(true);
|
| 34 |
+
setError(null);
|
| 35 |
+
startInterview(mode, Math.floor(Math.random() * 100000))
|
| 36 |
+
.then(setSession)
|
| 37 |
+
.catch((e: Error) => setError(e.message))
|
| 38 |
+
.finally(() => setBusy(false));
|
| 39 |
+
};
|
| 40 |
+
|
| 41 |
+
const submit = () => {
|
| 42 |
+
if (!session || !answer.trim()) return;
|
| 43 |
+
setBusy(true);
|
| 44 |
+
setError(null);
|
| 45 |
+
answerInterview(session.session_id, answer)
|
| 46 |
+
.then((s) => {
|
| 47 |
+
setSession(s);
|
| 48 |
+
setAnswer('');
|
| 49 |
+
})
|
| 50 |
+
.catch((e: Error) => setError(e.message))
|
| 51 |
+
.finally(() => setBusy(false));
|
| 52 |
+
};
|
| 53 |
+
|
| 54 |
+
const finish = () => {
|
| 55 |
+
if (!session) return;
|
| 56 |
+
setBusy(true);
|
| 57 |
+
endInterview(session.session_id)
|
| 58 |
+
.then(setSession)
|
| 59 |
+
.catch((e: Error) => setError(e.message))
|
| 60 |
+
.finally(() => setBusy(false));
|
| 61 |
+
};
|
| 62 |
+
|
| 63 |
+
return (
|
| 64 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 980 }}>
|
| 65 |
+
{error ? <ErrorNote error={error} /> : null}
|
| 66 |
+
|
| 67 |
+
{!session ? (
|
| 68 |
+
<SectionCard
|
| 69 |
+
title="AI interviewer"
|
| 70 |
+
help="No fixed question list: the next question is chosen from your last evaluation — weakness → targeted probe, strong answer → escalation or dimension switch, polished-but-shallow → depth probe. The transcript is saved as evidence."
|
| 71 |
+
>
|
| 72 |
+
<Typography variant="body2" sx={{ color: '#aab4be', mb: 2 }}>
|
| 73 |
+
Pick a mode and answer in your own words. The interviewer adapts after every answer.
|
| 74 |
+
</Typography>
|
| 75 |
+
<Box sx={{ display: 'flex', gap: 2, alignItems: 'center' }}>
|
| 76 |
+
<ToggleButtonGroup size="small" exclusive value={mode} onChange={(_e, v: string | null) => v && setMode(v)}>
|
| 77 |
+
<ToggleButton value="technical">Technical</ToggleButton>
|
| 78 |
+
<ToggleButton value="management">Management</ToggleButton>
|
| 79 |
+
<ToggleButton value="hybrid">Hybrid EM</ToggleButton>
|
| 80 |
+
</ToggleButtonGroup>
|
| 81 |
+
<Button variant="contained" onClick={start} disabled={busy}>
|
| 82 |
+
Start interview
|
| 83 |
+
</Button>
|
| 84 |
+
</Box>
|
| 85 |
+
</SectionCard>
|
| 86 |
+
) : null}
|
| 87 |
+
|
| 88 |
+
{session ? (
|
| 89 |
+
<SectionCard
|
| 90 |
+
title={`Interview (${session.mode})`}
|
| 91 |
+
action={
|
| 92 |
+
session.status === 'active' ? (
|
| 93 |
+
<Button size="small" color="warning" onClick={finish} disabled={busy}>
|
| 94 |
+
End & save transcript
|
| 95 |
+
</Button>
|
| 96 |
+
) : (
|
| 97 |
+
<Chip size="small" label="transcript saved as evidence" sx={{ bgcolor: '#00695c', color: '#80cbc4' }} />
|
| 98 |
+
)
|
| 99 |
+
}
|
| 100 |
+
>
|
| 101 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1.5 }}>
|
| 102 |
+
{session.turns.map((t, i) => (
|
| 103 |
+
<Box key={i}>
|
| 104 |
+
<Box
|
| 105 |
+
sx={{
|
| 106 |
+
bgcolor: '#12171d',
|
| 107 |
+
border: '1px solid #232a31',
|
| 108 |
+
borderLeft: `3px solid ${TYPE_LABEL[t.question_type].color}`,
|
| 109 |
+
borderRadius: 1,
|
| 110 |
+
p: 1.25,
|
| 111 |
+
maxWidth: '85%',
|
| 112 |
+
}}
|
| 113 |
+
>
|
| 114 |
+
<Typography variant="caption" sx={{ color: TYPE_LABEL[t.question_type].color, fontWeight: 700 }}>
|
| 115 |
+
INTERVIEWER · {TYPE_LABEL[t.question_type].label} · {t.competency_id} · difficulty {t.difficulty}
|
| 116 |
+
</Typography>
|
| 117 |
+
<Typography variant="body2" sx={{ color: '#e6e9ec', lineHeight: 1.6 }}>
|
| 118 |
+
{t.question}
|
| 119 |
+
</Typography>
|
| 120 |
+
</Box>
|
| 121 |
+
{t.answer ? (
|
| 122 |
+
<Box sx={{ display: 'flex', justifyContent: 'flex-end', mt: 1 }}>
|
| 123 |
+
<Box sx={{ bgcolor: '#1a2733', borderRadius: 1, p: 1.25, maxWidth: '85%' }}>
|
| 124 |
+
<Typography variant="body2" sx={{ color: '#cfd8dc' }}>
|
| 125 |
+
{t.answer}
|
| 126 |
+
</Typography>
|
| 127 |
+
{t.evaluation ? (
|
| 128 |
+
<Box sx={{ display: 'flex', gap: 0.75, mt: 1, flexWrap: 'wrap', alignItems: 'center' }}>
|
| 129 |
+
<ScoreChip score={t.evaluation.score} />
|
| 130 |
+
{t.evaluation.weaknesses.slice(0, 2).map((w, j) => (
|
| 131 |
+
<Chip key={j} size="small" label={`gap: ${w.slice(0, 44)}`} sx={{ bgcolor: '#232a31', color: '#ffcc80', fontSize: 10, height: 18 }} />
|
| 132 |
+
))}
|
| 133 |
+
{t.evaluation.evidence.length > 0 ? (
|
| 134 |
+
<Chip size="small" label={`${t.evaluation.evidence.length} evidence quote(s)`} sx={{ bgcolor: '#232a31', color: scoreColor(t.evaluation.score), fontSize: 10, height: 18 }} />
|
| 135 |
+
) : null}
|
| 136 |
+
</Box>
|
| 137 |
+
) : null}
|
| 138 |
+
</Box>
|
| 139 |
+
</Box>
|
| 140 |
+
) : null}
|
| 141 |
+
</Box>
|
| 142 |
+
))}
|
| 143 |
+
</Box>
|
| 144 |
+
|
| 145 |
+
{session.status === 'active' ? (
|
| 146 |
+
<Box sx={{ mt: 2 }}>
|
| 147 |
+
<TextField
|
| 148 |
+
multiline
|
| 149 |
+
minRows={3}
|
| 150 |
+
fullWidth
|
| 151 |
+
placeholder="Your answer…"
|
| 152 |
+
value={answer}
|
| 153 |
+
onChange={(e) => setAnswer(e.target.value)}
|
| 154 |
+
disabled={busy}
|
| 155 |
+
/>
|
| 156 |
+
<Box sx={{ display: 'flex', justifyContent: 'flex-end', mt: 1 }}>
|
| 157 |
+
<Button variant="contained" onClick={submit} disabled={busy || !answer.trim()}>
|
| 158 |
+
{busy ? 'Evaluating…' : 'Answer'}
|
| 159 |
+
</Button>
|
| 160 |
+
</Box>
|
| 161 |
+
</Box>
|
| 162 |
+
) : (
|
| 163 |
+
<Alert severity="success" variant="outlined" sx={{ mt: 2 }}>
|
| 164 |
+
Interview complete. Answers were folded into your readiness matrix and the transcript
|
| 165 |
+
stored in the evidence library.
|
| 166 |
+
<Button size="small" sx={{ ml: 1 }} onClick={start}>
|
| 167 |
+
New interview
|
| 168 |
+
</Button>
|
| 169 |
+
</Alert>
|
| 170 |
+
)}
|
| 171 |
+
</SectionCard>
|
| 172 |
+
) : null}
|
| 173 |
+
</Box>
|
| 174 |
+
);
|
| 175 |
+
}
|
src/pages/hillclimb/MatrixView.tsx
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Competency matrix: explainable scores — click a row to see the evidence behind it. */
|
| 2 |
+
|
| 3 |
+
import { useState } from 'react';
|
| 4 |
+
import Box from '@mui/material/Box';
|
| 5 |
+
import Button from '@mui/material/Button';
|
| 6 |
+
import Chip from '@mui/material/Chip';
|
| 7 |
+
import Dialog from '@mui/material/Dialog';
|
| 8 |
+
import DialogActions from '@mui/material/DialogActions';
|
| 9 |
+
import DialogContent from '@mui/material/DialogContent';
|
| 10 |
+
import DialogTitle from '@mui/material/DialogTitle';
|
| 11 |
+
import Table from '@mui/material/Table';
|
| 12 |
+
import TableBody from '@mui/material/TableBody';
|
| 13 |
+
import TableCell from '@mui/material/TableCell';
|
| 14 |
+
import TableHead from '@mui/material/TableHead';
|
| 15 |
+
import TableRow from '@mui/material/TableRow';
|
| 16 |
+
import Typography from '@mui/material/Typography';
|
| 17 |
+
import { ErrorNote, LoadingBox, SectionCard } from '../../components/labeleval/shared';
|
| 18 |
+
import { ReadinessChip, useAsync } from '../../components/hillclimb/shared';
|
| 19 |
+
import { getEvidence, getReadiness } from '../../services/hillclimb';
|
| 20 |
+
import type { EvidenceItem, ReadinessRow } from '../../types/hillclimb';
|
| 21 |
+
import type { GoFn } from './HillClimbSection';
|
| 22 |
+
|
| 23 |
+
function fmtScore(v: number): string {
|
| 24 |
+
return v > 0 ? v.toFixed(1) : '—';
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
export default function MatrixView({ go }: { go: GoFn }) {
|
| 28 |
+
const readiness = useAsync(getReadiness);
|
| 29 |
+
const [selected, setSelected] = useState<ReadinessRow | null>(null);
|
| 30 |
+
const [evidence, setEvidence] = useState<EvidenceItem[] | null>(null);
|
| 31 |
+
|
| 32 |
+
const openRow = (row: ReadinessRow) => {
|
| 33 |
+
setSelected(row);
|
| 34 |
+
setEvidence(null);
|
| 35 |
+
getEvidence(row.competency_id)
|
| 36 |
+
.then((r) => setEvidence(r.evidence))
|
| 37 |
+
.catch(() => setEvidence([]));
|
| 38 |
+
};
|
| 39 |
+
|
| 40 |
+
if (readiness.loading && !readiness.data) return <LoadingBox label="Loading matrix…" />;
|
| 41 |
+
if (readiness.error && !readiness.data) return <ErrorNote error={readiness.error} />;
|
| 42 |
+
|
| 43 |
+
const rows = readiness.data?.matrix ?? [];
|
| 44 |
+
const bottleneckId = readiness.data?.bottleneck?.competency_id;
|
| 45 |
+
|
| 46 |
+
return (
|
| 47 |
+
<SectionCard
|
| 48 |
+
title={`Competency matrix (${rows.length})`}
|
| 49 |
+
help="Knowledge / application / evidence are tracked separately per competency and never collapsed into one number. Click a row to see the evidence behind its scores. The flagged row is your current bottleneck."
|
| 50 |
+
>
|
| 51 |
+
<Table size="small">
|
| 52 |
+
<TableHead>
|
| 53 |
+
<TableRow>
|
| 54 |
+
<TableCell>Competency</TableCell>
|
| 55 |
+
<TableCell>Phase</TableCell>
|
| 56 |
+
<TableCell>Dimension</TableCell>
|
| 57 |
+
<TableCell align="right">Knowledge</TableCell>
|
| 58 |
+
<TableCell align="right">Application</TableCell>
|
| 59 |
+
<TableCell align="right">Evidence</TableCell>
|
| 60 |
+
<TableCell>State</TableCell>
|
| 61 |
+
</TableRow>
|
| 62 |
+
</TableHead>
|
| 63 |
+
<TableBody>
|
| 64 |
+
{rows.map((r) => (
|
| 65 |
+
<TableRow
|
| 66 |
+
key={r.competency_id}
|
| 67 |
+
hover
|
| 68 |
+
sx={{ cursor: 'pointer', bgcolor: r.competency_id === bottleneckId ? 'rgba(230,81,0,0.12)' : undefined }}
|
| 69 |
+
onClick={() => openRow(r)}
|
| 70 |
+
>
|
| 71 |
+
<TableCell>
|
| 72 |
+
<Typography variant="body2" sx={{ fontWeight: 600 }}>
|
| 73 |
+
{r.name}
|
| 74 |
+
{r.competency_id === bottleneckId ? (
|
| 75 |
+
<Chip size="small" label="BOTTLENECK" sx={{ ml: 1, height: 16, fontSize: 9, bgcolor: '#e65100', color: '#ffe0b2', fontWeight: 800 }} />
|
| 76 |
+
) : null}
|
| 77 |
+
</Typography>
|
| 78 |
+
<Typography variant="caption" sx={{ color: '#5c6873', fontFamily: 'monospace' }}>
|
| 79 |
+
{r.competency_id}
|
| 80 |
+
</Typography>
|
| 81 |
+
</TableCell>
|
| 82 |
+
<TableCell>{r.phase}</TableCell>
|
| 83 |
+
<TableCell>
|
| 84 |
+
<Typography variant="caption" sx={{ color: '#8a949e' }}>
|
| 85 |
+
{r.dimension}
|
| 86 |
+
</Typography>
|
| 87 |
+
</TableCell>
|
| 88 |
+
<TableCell align="right" sx={{ fontFamily: 'monospace', fontSize: 12 }}>{fmtScore(r.knowledge_score)}</TableCell>
|
| 89 |
+
<TableCell align="right" sx={{ fontFamily: 'monospace', fontSize: 12 }}>{fmtScore(r.application_score)}</TableCell>
|
| 90 |
+
<TableCell align="right" sx={{ fontFamily: 'monospace', fontSize: 12 }}>{fmtScore(r.evidence_score)}</TableCell>
|
| 91 |
+
<TableCell>
|
| 92 |
+
<ReadinessChip state={r.readiness_state} />
|
| 93 |
+
</TableCell>
|
| 94 |
+
</TableRow>
|
| 95 |
+
))}
|
| 96 |
+
</TableBody>
|
| 97 |
+
</Table>
|
| 98 |
+
|
| 99 |
+
<Dialog open={selected !== null} onClose={() => setSelected(null)} maxWidth="sm" fullWidth>
|
| 100 |
+
<DialogTitle>
|
| 101 |
+
{selected?.name}
|
| 102 |
+
<Typography variant="caption" sx={{ display: 'block', color: '#8a949e', fontFamily: 'monospace' }}>
|
| 103 |
+
{selected?.competency_id} · prerequisites: {selected?.prerequisites.join(', ') || 'none'}
|
| 104 |
+
</Typography>
|
| 105 |
+
</DialogTitle>
|
| 106 |
+
<DialogContent>
|
| 107 |
+
{selected ? (
|
| 108 |
+
<>
|
| 109 |
+
<Typography variant="body2" sx={{ mb: 1.5 }}>
|
| 110 |
+
knowledge <b>{fmtScore(selected.knowledge_score)}</b> �� application{' '}
|
| 111 |
+
<b>{fmtScore(selected.application_score)}</b> · evidence{' '}
|
| 112 |
+
<b>{fmtScore(selected.evidence_score)}</b> — <ReadinessChip state={selected.readiness_state} />
|
| 113 |
+
</Typography>
|
| 114 |
+
<Typography variant="caption" sx={{ color: '#4fc3f7', fontWeight: 700 }}>
|
| 115 |
+
EVIDENCE BEHIND THIS SCORE
|
| 116 |
+
</Typography>
|
| 117 |
+
{evidence === null ? (
|
| 118 |
+
<LoadingBox label="Loading evidence…" />
|
| 119 |
+
) : evidence.length === 0 ? (
|
| 120 |
+
<Typography variant="body2" sx={{ color: '#8a949e', mt: 1 }}>
|
| 121 |
+
No evidence yet — every score requires evidence, which is why this competency is
|
| 122 |
+
still {selected.readiness_state.replace(/_/g, ' ')}.
|
| 123 |
+
</Typography>
|
| 124 |
+
) : (
|
| 125 |
+
evidence.map((e) => (
|
| 126 |
+
<Box key={e.evidence_id} sx={{ my: 1, p: 1, bgcolor: '#12171d', borderRadius: 1, border: '1px solid #232a31' }}>
|
| 127 |
+
<Typography variant="caption" sx={{ color: '#aab4be', fontWeight: 700 }}>
|
| 128 |
+
{e.artifact_type.replace(/_/g, ' ')} · score {e.score}/5 · {new Date(e.timestamp).toLocaleString()}
|
| 129 |
+
</Typography>
|
| 130 |
+
<Typography variant="body2" sx={{ color: '#cfd8dc' }}>
|
| 131 |
+
{e.summary}
|
| 132 |
+
</Typography>
|
| 133 |
+
{e.quotes.slice(0, 2).map((q, i) => (
|
| 134 |
+
<Typography key={i} variant="caption" sx={{ display: 'block', fontStyle: 'italic', color: '#8a949e' }}>
|
| 135 |
+
“{q}”
|
| 136 |
+
</Typography>
|
| 137 |
+
))}
|
| 138 |
+
</Box>
|
| 139 |
+
))
|
| 140 |
+
)}
|
| 141 |
+
</>
|
| 142 |
+
) : null}
|
| 143 |
+
</DialogContent>
|
| 144 |
+
<DialogActions>
|
| 145 |
+
<Button onClick={() => go('practice', null, selected?.competency_id ?? null)}>Practice this</Button>
|
| 146 |
+
<Button onClick={() => setSelected(null)}>Close</Button>
|
| 147 |
+
</DialogActions>
|
| 148 |
+
</Dialog>
|
| 149 |
+
</SectionCard>
|
| 150 |
+
);
|
| 151 |
+
}
|
src/pages/hillclimb/PracticeView.tsx
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Practice workspace: scenario + free-text answer -> structured coaching panel. */
|
| 2 |
+
|
| 3 |
+
import { useEffect, useState } from 'react';
|
| 4 |
+
import Alert from '@mui/material/Alert';
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Button from '@mui/material/Button';
|
| 7 |
+
import Chip from '@mui/material/Chip';
|
| 8 |
+
import FormControlLabel from '@mui/material/FormControlLabel';
|
| 9 |
+
import MenuItem from '@mui/material/MenuItem';
|
| 10 |
+
import Switch from '@mui/material/Switch';
|
| 11 |
+
import TextField from '@mui/material/TextField';
|
| 12 |
+
import Typography from '@mui/material/Typography';
|
| 13 |
+
import { RefreshCw } from 'lucide-react';
|
| 14 |
+
import { ErrorNote, SectionCard } from '../../components/labeleval/shared';
|
| 15 |
+
import { EvaluationPanel, useAsync } from '../../components/hillclimb/shared';
|
| 16 |
+
import { generateExercise, getBlueprint, submitExercise } from '../../services/hillclimb';
|
| 17 |
+
import type { Exercise, SubmitExerciseResponse } from '../../types/hillclimb';
|
| 18 |
+
import type { GoFn } from './HillClimbSection';
|
| 19 |
+
|
| 20 |
+
export default function PracticeView({
|
| 21 |
+
go,
|
| 22 |
+
seedExercise,
|
| 23 |
+
seedCompetency,
|
| 24 |
+
}: {
|
| 25 |
+
go: GoFn;
|
| 26 |
+
seedExercise: Exercise | null;
|
| 27 |
+
seedCompetency: string | null;
|
| 28 |
+
}) {
|
| 29 |
+
const blueprint = useAsync(getBlueprint);
|
| 30 |
+
const [competencyId, setCompetencyId] = useState<string>(
|
| 31 |
+
seedExercise?.competency_id ?? seedCompetency ?? 'p1.regression_detection'
|
| 32 |
+
);
|
| 33 |
+
const [difficulty, setDifficulty] = useState(2);
|
| 34 |
+
const [exercise, setExercise] = useState<Exercise | null>(seedExercise);
|
| 35 |
+
const [answer, setAnswer] = useState('');
|
| 36 |
+
const [result, setResult] = useState<SubmitExerciseResponse | null>(null);
|
| 37 |
+
const [asAssessment, setAsAssessment] = useState(false);
|
| 38 |
+
const [busy, setBusy] = useState(false);
|
| 39 |
+
const [error, setError] = useState<string | null>(null);
|
| 40 |
+
|
| 41 |
+
const fetchExercise = (cid: string, diff: number) => {
|
| 42 |
+
setBusy(true);
|
| 43 |
+
setError(null);
|
| 44 |
+
setResult(null);
|
| 45 |
+
setAnswer('');
|
| 46 |
+
generateExercise(cid, diff)
|
| 47 |
+
.then(setExercise)
|
| 48 |
+
.catch((e: Error) => setError(e.message))
|
| 49 |
+
.finally(() => setBusy(false));
|
| 50 |
+
};
|
| 51 |
+
|
| 52 |
+
useEffect(() => {
|
| 53 |
+
if (!seedExercise) fetchExercise(competencyId, difficulty);
|
| 54 |
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 55 |
+
}, []);
|
| 56 |
+
|
| 57 |
+
const submit = () => {
|
| 58 |
+
if (!exercise || !answer.trim()) return;
|
| 59 |
+
setBusy(true);
|
| 60 |
+
setError(null);
|
| 61 |
+
submitExercise(exercise.exercise_id, answer, asAssessment)
|
| 62 |
+
.then(setResult)
|
| 63 |
+
.catch((e: Error) => setError(e.message))
|
| 64 |
+
.finally(() => setBusy(false));
|
| 65 |
+
};
|
| 66 |
+
|
| 67 |
+
const continueWithFollowUp = () => {
|
| 68 |
+
if (!result || !exercise) return;
|
| 69 |
+
const probe = result.evaluation.follow_up_question;
|
| 70 |
+
setExercise({ ...exercise, scenario: probe, exercise_id: exercise.exercise_id });
|
| 71 |
+
setAnswer('');
|
| 72 |
+
setResult(null);
|
| 73 |
+
};
|
| 74 |
+
|
| 75 |
+
return (
|
| 76 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap', alignItems: 'flex-start' }}>
|
| 77 |
+
<Box sx={{ flex: '3 1 480px', display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 78 |
+
<SectionCard
|
| 79 |
+
title="Exercise"
|
| 80 |
+
action={
|
| 81 |
+
<Box sx={{ display: 'flex', gap: 1, alignItems: 'center' }}>
|
| 82 |
+
<TextField
|
| 83 |
+
select
|
| 84 |
+
size="small"
|
| 85 |
+
label="Competency"
|
| 86 |
+
value={competencyId}
|
| 87 |
+
onChange={(e) => {
|
| 88 |
+
setCompetencyId(e.target.value);
|
| 89 |
+
fetchExercise(e.target.value, difficulty);
|
| 90 |
+
}}
|
| 91 |
+
sx={{ minWidth: 240 }}
|
| 92 |
+
>
|
| 93 |
+
{(blueprint.data?.competencies ?? []).map((c) => (
|
| 94 |
+
<MenuItem key={c.id} value={c.id}>
|
| 95 |
+
P{c.phase} · {c.name}
|
| 96 |
+
</MenuItem>
|
| 97 |
+
))}
|
| 98 |
+
</TextField>
|
| 99 |
+
<TextField
|
| 100 |
+
select
|
| 101 |
+
size="small"
|
| 102 |
+
label="Difficulty"
|
| 103 |
+
value={difficulty}
|
| 104 |
+
onChange={(e) => {
|
| 105 |
+
const d = Number(e.target.value);
|
| 106 |
+
setDifficulty(d);
|
| 107 |
+
fetchExercise(competencyId, d);
|
| 108 |
+
}}
|
| 109 |
+
sx={{ width: 110 }}
|
| 110 |
+
>
|
| 111 |
+
<MenuItem value={1}>Warmup</MenuItem>
|
| 112 |
+
<MenuItem value={2}>Core</MenuItem>
|
| 113 |
+
<MenuItem value={3}>Stretch</MenuItem>
|
| 114 |
+
</TextField>
|
| 115 |
+
<Button
|
| 116 |
+
size="small"
|
| 117 |
+
startIcon={<RefreshCw size={14} />}
|
| 118 |
+
onClick={() => fetchExercise(competencyId, difficulty)}
|
| 119 |
+
disabled={busy}
|
| 120 |
+
>
|
| 121 |
+
New variant
|
| 122 |
+
</Button>
|
| 123 |
+
</Box>
|
| 124 |
+
}
|
| 125 |
+
help="Every regeneration produces a structurally different scenario for the same competency (seeded template variation) — repetition can't be memorized. Answers are scored on concept coverage, tradeoffs, and quantified claims; never on length."
|
| 126 |
+
>
|
| 127 |
+
{error ? <ErrorNote error={error} /> : null}
|
| 128 |
+
{exercise ? (
|
| 129 |
+
<>
|
| 130 |
+
<Box sx={{ display: 'flex', gap: 0.75, mb: 1.5, flexWrap: 'wrap' }}>
|
| 131 |
+
<Chip size="small" label={exercise.competency_id} sx={{ bgcolor: '#232a31', fontFamily: 'monospace', fontSize: 11 }} />
|
| 132 |
+
<Chip size="small" label={`family: ${exercise.family}`} sx={{ bgcolor: '#232a31', fontSize: 11 }} />
|
| 133 |
+
<Chip size="small" label={exercise.template_id} sx={{ bgcolor: '#232a31', fontFamily: 'monospace', fontSize: 11 }} />
|
| 134 |
+
</Box>
|
| 135 |
+
<Typography variant="body2" sx={{ color: '#e6e9ec', lineHeight: 1.7, mb: 2 }}>
|
| 136 |
+
{exercise.scenario}
|
| 137 |
+
</Typography>
|
| 138 |
+
{exercise.linked_tool ? (
|
| 139 |
+
<Alert severity="info" variant="outlined" sx={{ mb: 2, py: 0.25 }}>
|
| 140 |
+
{exercise.linked_tool.label} — open the{' '}
|
| 141 |
+
<a href={`#${exercise.linked_tool.page}`} style={{ color: '#4fc3f7' }}>
|
| 142 |
+
RCA workbench
|
| 143 |
+
</a>{' '}
|
| 144 |
+
to run this diagnosis against live data.
|
| 145 |
+
</Alert>
|
| 146 |
+
) : null}
|
| 147 |
+
<TextField
|
| 148 |
+
multiline
|
| 149 |
+
minRows={6}
|
| 150 |
+
fullWidth
|
| 151 |
+
placeholder="Your answer — enumerate causes, mechanisms, numbers, tradeoffs, and a verification plan…"
|
| 152 |
+
value={answer}
|
| 153 |
+
onChange={(e) => setAnswer(e.target.value)}
|
| 154 |
+
disabled={busy}
|
| 155 |
+
/>
|
| 156 |
+
<Box sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', mt: 1.5 }}>
|
| 157 |
+
<FormControlLabel
|
| 158 |
+
control={
|
| 159 |
+
<Switch size="small" checked={asAssessment} onChange={(e) => setAsAssessment(e.target.checked)} />
|
| 160 |
+
}
|
| 161 |
+
label={
|
| 162 |
+
<Typography variant="caption" sx={{ color: '#8a949e' }}>
|
| 163 |
+
Submit as assessment (4+/5 passes; failing routes remediation to the diagnosed prerequisite)
|
| 164 |
+
</Typography>
|
| 165 |
+
}
|
| 166 |
+
/>
|
| 167 |
+
<Button variant="contained" onClick={submit} disabled={busy || !answer.trim()}>
|
| 168 |
+
{busy ? 'Evaluating…' : 'Submit for coaching'}
|
| 169 |
+
</Button>
|
| 170 |
+
</Box>
|
| 171 |
+
</>
|
| 172 |
+
) : null}
|
| 173 |
+
</SectionCard>
|
| 174 |
+
|
| 175 |
+
{result ? (
|
| 176 |
+
<SectionCard
|
| 177 |
+
title="Coaching"
|
| 178 |
+
action={
|
| 179 |
+
result.evaluation.follow_up_question ? (
|
| 180 |
+
<Button size="small" variant="outlined" onClick={continueWithFollowUp}>
|
| 181 |
+
Continue with follow-up
|
| 182 |
+
</Button>
|
| 183 |
+
) : undefined
|
| 184 |
+
}
|
| 185 |
+
>
|
| 186 |
+
<EvaluationPanel evaluation={result.evaluation} coaching={result.coaching} />
|
| 187 |
+
{result.evaluation.follow_up_question ? (
|
| 188 |
+
<Alert severity="info" variant="outlined" sx={{ mt: 2, py: 0.25 }}>
|
| 189 |
+
<b>Follow-up:</b> {result.evaluation.follow_up_question}
|
| 190 |
+
</Alert>
|
| 191 |
+
) : null}
|
| 192 |
+
{result.journey && result.journey.state === 'REMEDIATION' ? (
|
| 193 |
+
<Alert severity="warning" variant="outlined" sx={{ mt: 1.5, py: 0.25 }}>
|
| 194 |
+
Assessment failed — remediation routed to prerequisite{' '}
|
| 195 |
+
<b>{result.journey.remediation_target}</b>.{' '}
|
| 196 |
+
<Button size="small" onClick={() => go('practice', null, result.journey!.remediation_target)}>
|
| 197 |
+
Practice it now
|
| 198 |
+
</Button>
|
| 199 |
+
</Alert>
|
| 200 |
+
) : null}
|
| 201 |
+
</SectionCard>
|
| 202 |
+
) : null}
|
| 203 |
+
</Box>
|
| 204 |
+
|
| 205 |
+
<Box sx={{ flex: '1 1 260px', display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 206 |
+
{result?.readiness_for_competency ? (
|
| 207 |
+
<SectionCard title="Competency readiness" help="Updated after every attempt. Knowledge, application and evidence are tracked separately.">
|
| 208 |
+
<Typography variant="caption" sx={{ color: '#8a949e', display: 'block' }}>
|
| 209 |
+
knowledge {result.readiness_for_competency.knowledge_score.toFixed(1)} · application{' '}
|
| 210 |
+
{result.readiness_for_competency.application_score.toFixed(1)} · evidence{' '}
|
| 211 |
+
{result.readiness_for_competency.evidence_score.toFixed(1)}
|
| 212 |
+
</Typography>
|
| 213 |
+
<Chip
|
| 214 |
+
size="small"
|
| 215 |
+
label={result.readiness_for_competency.readiness_state.replace(/_/g, ' ')}
|
| 216 |
+
sx={{ mt: 1, bgcolor: '#232a31', color: '#aab4be', fontWeight: 700 }}
|
| 217 |
+
/>
|
| 218 |
+
</SectionCard>
|
| 219 |
+
) : null}
|
| 220 |
+
{exercise ? (
|
| 221 |
+
<SectionCard title="Common failure modes" help="How answers to this exercise family typically go wrong.">
|
| 222 |
+
{exercise.common_failure_modes.map((f, i) => (
|
| 223 |
+
<Typography key={i} variant="caption" sx={{ color: '#8a949e', display: 'block', mb: 0.75 }}>
|
| 224 |
+
• {f}
|
| 225 |
+
</Typography>
|
| 226 |
+
))}
|
| 227 |
+
</SectionCard>
|
| 228 |
+
) : null}
|
| 229 |
+
</Box>
|
| 230 |
+
</Box>
|
| 231 |
+
);
|
| 232 |
+
}
|
src/pages/hillclimb/SimulationView.tsx
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Hill-climbing simulation: state gauges, hypothesis input, intervention picker, event log, debrief. */
|
| 2 |
+
|
| 3 |
+
import { useState } from 'react';
|
| 4 |
+
import Alert from '@mui/material/Alert';
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Button from '@mui/material/Button';
|
| 7 |
+
import Checkbox from '@mui/material/Checkbox';
|
| 8 |
+
import Chip from '@mui/material/Chip';
|
| 9 |
+
import FormControlLabel from '@mui/material/FormControlLabel';
|
| 10 |
+
import MenuItem from '@mui/material/MenuItem';
|
| 11 |
+
import TextField from '@mui/material/TextField';
|
| 12 |
+
import Typography from '@mui/material/Typography';
|
| 13 |
+
import { ErrorNote, HBar, LoadingBox, SectionCard } from '../../components/labeleval/shared';
|
| 14 |
+
import { useAsync } from '../../components/hillclimb/shared';
|
| 15 |
+
import { getSimulationCatalog, startSimulation, stepSimulation } from '../../services/hillclimb';
|
| 16 |
+
import type { SimulationState } from '../../types/hillclimb';
|
| 17 |
+
|
| 18 |
+
function metricColor(name: string, value: number, inverted: string[]): string {
|
| 19 |
+
const effective = inverted.includes(name) ? 100 - value : value;
|
| 20 |
+
if (effective >= 60) return '#66bb6a';
|
| 21 |
+
if (effective >= 40) return '#f9a825';
|
| 22 |
+
return '#ef5350';
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
export default function SimulationView() {
|
| 26 |
+
const catalog = useAsync(getSimulationCatalog);
|
| 27 |
+
const [sim, setSim] = useState<SimulationState | null>(null);
|
| 28 |
+
const [hypothesis, setHypothesis] = useState('');
|
| 29 |
+
const [interventionId, setInterventionId] = useState('add_monitoring');
|
| 30 |
+
const [revert, setRevert] = useState(false);
|
| 31 |
+
const [busy, setBusy] = useState(false);
|
| 32 |
+
const [error, setError] = useState<string | null>(null);
|
| 33 |
+
|
| 34 |
+
const start = () => {
|
| 35 |
+
setBusy(true);
|
| 36 |
+
setError(null);
|
| 37 |
+
startSimulation(Math.floor(Math.random() * 100000), 8)
|
| 38 |
+
.then(setSim)
|
| 39 |
+
.catch((e: Error) => setError(e.message))
|
| 40 |
+
.finally(() => setBusy(false));
|
| 41 |
+
};
|
| 42 |
+
|
| 43 |
+
const step = () => {
|
| 44 |
+
if (!sim || !hypothesis.trim()) return;
|
| 45 |
+
setBusy(true);
|
| 46 |
+
setError(null);
|
| 47 |
+
stepSimulation(sim.sim_id, hypothesis, interventionId, revert)
|
| 48 |
+
.then((s) => {
|
| 49 |
+
setSim(s);
|
| 50 |
+
setHypothesis('');
|
| 51 |
+
setRevert(false);
|
| 52 |
+
})
|
| 53 |
+
.catch((e: Error) => setError(e.message))
|
| 54 |
+
.finally(() => setBusy(false));
|
| 55 |
+
};
|
| 56 |
+
|
| 57 |
+
if (catalog.loading && !catalog.data) return <LoadingBox label="Loading scenario…" />;
|
| 58 |
+
if (catalog.error && !catalog.data) return <ErrorNote error={catalog.error} />;
|
| 59 |
+
|
| 60 |
+
const inverted = catalog.data?.inverted_metrics ?? [];
|
| 61 |
+
const interventions = catalog.data?.interventions ?? [];
|
| 62 |
+
const scenario = catalog.data?.scenarios[0];
|
| 63 |
+
const lastTurn = sim?.history[sim.history.length - 1] ?? null;
|
| 64 |
+
|
| 65 |
+
return (
|
| 66 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 67 |
+
{error ? <ErrorNote error={error} /> : null}
|
| 68 |
+
|
| 69 |
+
{!sim ? (
|
| 70 |
+
<SectionCard title={scenario?.title ?? 'Scenario'} help="A stateful multi-objective exercise: each turn you write a falsifiable hypothesis, pick one intervention, measure the seeded (deterministic per seed) effect, and keep or reject. Safety has a hard floor — breach it and an incident fires.">
|
| 71 |
+
<Typography variant="body2" sx={{ color: '#e6e9ec', lineHeight: 1.7, mb: 2 }}>
|
| 72 |
+
{scenario?.narrative}
|
| 73 |
+
</Typography>
|
| 74 |
+
<Button variant="contained" onClick={start} disabled={busy}>
|
| 75 |
+
Take over the team (8 turns)
|
| 76 |
+
</Button>
|
| 77 |
+
</SectionCard>
|
| 78 |
+
) : null}
|
| 79 |
+
|
| 80 |
+
{sim ? (
|
| 81 |
+
<>
|
| 82 |
+
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1, flexWrap: 'wrap' }}>
|
| 83 |
+
<Chip size="small" label={`turn ${sim.turn}/${sim.max_turns}`} sx={{ bgcolor: '#0d47a1', color: '#90caf9', fontWeight: 700 }} />
|
| 84 |
+
<Chip
|
| 85 |
+
size="small"
|
| 86 |
+
label={`balanced objective ${sim.objective_history[sim.objective_history.length - 1].toFixed(1)}`}
|
| 87 |
+
sx={{ bgcolor: '#232a31', color: '#e6e9ec', fontFamily: 'monospace' }}
|
| 88 |
+
/>
|
| 89 |
+
{lastTurn ? (
|
| 90 |
+
<Chip
|
| 91 |
+
size="small"
|
| 92 |
+
label={`Δ ${lastTurn.objective_delta >= 0 ? '+' : ''}${lastTurn.objective_delta.toFixed(1)} → ${
|
| 93 |
+
lastTurn.verdict === 'keep_recommended' ? 'KEEP' : 'REJECT'
|
| 94 |
+
}`}
|
| 95 |
+
sx={{
|
| 96 |
+
bgcolor: lastTurn.objective_delta >= 0 ? '#1b5e20' : '#5d1414',
|
| 97 |
+
color: lastTurn.objective_delta >= 0 ? '#a5d6a7' : '#ffcdd2',
|
| 98 |
+
fontWeight: 700,
|
| 99 |
+
}}
|
| 100 |
+
/>
|
| 101 |
+
) : null}
|
| 102 |
+
{sim.status === 'complete' ? (
|
| 103 |
+
<Chip size="small" label="COMPLETE — see debrief below" sx={{ bgcolor: '#00695c', color: '#80cbc4', fontWeight: 700 }} />
|
| 104 |
+
) : null}
|
| 105 |
+
</Box>
|
| 106 |
+
|
| 107 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap', alignItems: 'flex-start' }}>
|
| 108 |
+
<SectionCard
|
| 109 |
+
title="Competing objectives"
|
| 110 |
+
help="All ten metrics matter; cost and risk are inverted (lower is better). Safety below the hard floor triggers an incident with second-order damage."
|
| 111 |
+
sx={{ flex: '1 1 360px' }}
|
| 112 |
+
>
|
| 113 |
+
{(catalog.data?.metrics ?? []).map((m) => (
|
| 114 |
+
<HBar
|
| 115 |
+
key={m}
|
| 116 |
+
label={`${m.replace(/_/g, ' ')}${inverted.includes(m) ? ' (lower=better)' : ''}`}
|
| 117 |
+
value={sim.metrics[m] ?? 0}
|
| 118 |
+
max={100}
|
| 119 |
+
color={metricColor(m, sim.metrics[m] ?? 0, inverted)}
|
| 120 |
+
valueLabel={(sim.metrics[m] ?? 0).toFixed(0)}
|
| 121 |
+
/>
|
| 122 |
+
))}
|
| 123 |
+
</SectionCard>
|
| 124 |
+
|
| 125 |
+
<Box sx={{ flex: '1 1 380px', display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 126 |
+
{sim.status === 'active' ? (
|
| 127 |
+
<SectionCard title={`Turn ${sim.turn + 1}: hypothesis → intervention`}>
|
| 128 |
+
<TextField
|
| 129 |
+
label="Hypothesis (falsifiable: metric + direction + magnitude)"
|
| 130 |
+
placeholder="e.g. 'Adding monitoring will raise safety by ~5 within 2 turns'"
|
| 131 |
+
multiline
|
| 132 |
+
minRows={2}
|
| 133 |
+
fullWidth
|
| 134 |
+
value={hypothesis}
|
| 135 |
+
onChange={(e) => setHypothesis(e.target.value)}
|
| 136 |
+
sx={{ mb: 1.5 }}
|
| 137 |
+
/>
|
| 138 |
+
<TextField
|
| 139 |
+
select
|
| 140 |
+
fullWidth
|
| 141 |
+
size="small"
|
| 142 |
+
label="Intervention"
|
| 143 |
+
value={interventionId}
|
| 144 |
+
onChange={(e) => setInterventionId(e.target.value)}
|
| 145 |
+
sx={{ mb: 1 }}
|
| 146 |
+
>
|
| 147 |
+
{interventions.map((i) => (
|
| 148 |
+
<MenuItem key={i.id} value={i.id}>
|
| 149 |
+
{i.label}
|
| 150 |
+
</MenuItem>
|
| 151 |
+
))}
|
| 152 |
+
</TextField>
|
| 153 |
+
<Typography variant="caption" sx={{ color: '#8a949e', display: 'block', mb: 1 }}>
|
| 154 |
+
{interventions.find((i) => i.id === interventionId)?.description}{' '}
|
| 155 |
+
<i>{interventions.find((i) => i.id === interventionId)?.second_order}</i>
|
| 156 |
+
</Typography>
|
| 157 |
+
{sim.history.length > 0 ? (
|
| 158 |
+
<FormControlLabel
|
| 159 |
+
control={<Checkbox size="small" checked={revert} onChange={(e) => setRevert(e.target.checked)} />}
|
| 160 |
+
label={
|
| 161 |
+
<Typography variant="caption" sx={{ color: '#ffb74d' }}>
|
| 162 |
+
Reject previous intervention first (roll back its immediate effects)
|
| 163 |
+
</Typography>
|
| 164 |
+
}
|
| 165 |
+
/>
|
| 166 |
+
) : null}
|
| 167 |
+
<Box sx={{ display: 'flex', justifyContent: 'flex-end' }}>
|
| 168 |
+
<Button variant="contained" onClick={step} disabled={busy || !hypothesis.trim()}>
|
| 169 |
+
{busy ? 'Measuring…' : 'Intervene & measure'}
|
| 170 |
+
</Button>
|
| 171 |
+
</Box>
|
| 172 |
+
</SectionCard>
|
| 173 |
+
) : null}
|
| 174 |
+
|
| 175 |
+
<SectionCard title="Event log" sx={{ maxHeight: 420, overflowY: 'auto' }}>
|
| 176 |
+
{sim.history.length === 0 ? (
|
| 177 |
+
<Typography variant="body2" sx={{ color: '#8a949e' }}>
|
| 178 |
+
No turns yet.
|
| 179 |
+
</Typography>
|
| 180 |
+
) : null}
|
| 181 |
+
{[...sim.history].reverse().map((t) => (
|
| 182 |
+
<Box key={t.turn} sx={{ mb: 1.5, pb: 1, borderBottom: '1px solid #232a31' }}>
|
| 183 |
+
<Typography variant="caption" sx={{ color: '#4fc3f7', fontWeight: 700 }}>
|
| 184 |
+
TURN {t.turn} · {t.intervention_id} · Δobj {t.objective_delta >= 0 ? '+' : ''}
|
| 185 |
+
{t.objective_delta.toFixed(1)}
|
| 186 |
+
{' '}
|
| 187 |
+
<Chip
|
| 188 |
+
size="small"
|
| 189 |
+
label={`hypothesis: ${t.hypothesis_assessment.quality}`}
|
| 190 |
+
sx={{
|
| 191 |
+
height: 16, fontSize: 9.5, ml: 0.5,
|
| 192 |
+
bgcolor: t.hypothesis_assessment.quality === 'strong' ? '#1b5e20' : t.hypothesis_assessment.quality === 'directional' ? '#f9a825' : '#5d1414',
|
| 193 |
+
color: t.hypothesis_assessment.quality === 'directional' ? '#212121' : '#e6e9ec',
|
| 194 |
+
}}
|
| 195 |
+
/>
|
| 196 |
+
</Typography>
|
| 197 |
+
<Typography variant="caption" sx={{ color: '#aab4be', display: 'block', fontStyle: 'italic' }}>
|
| 198 |
+
“{t.hypothesis}”
|
| 199 |
+
</Typography>
|
| 200 |
+
{t.delayed_landed.map((d, i) => (
|
| 201 |
+
<Typography key={i} variant="caption" sx={{ color: '#9ccc65', display: 'block' }}>
|
| 202 |
+
⏲ {d}
|
| 203 |
+
</Typography>
|
| 204 |
+
))}
|
| 205 |
+
{t.events.map((e, i) => (
|
| 206 |
+
<Typography key={i} variant="caption" sx={{ color: '#ef9a9a', display: 'block' }}>
|
| 207 |
+
⚠ {e}
|
| 208 |
+
</Typography>
|
| 209 |
+
))}
|
| 210 |
+
</Box>
|
| 211 |
+
))}
|
| 212 |
+
</SectionCard>
|
| 213 |
+
</Box>
|
| 214 |
+
</Box>
|
| 215 |
+
|
| 216 |
+
{sim.debrief ? (
|
| 217 |
+
<SectionCard title="Debrief — decisions mapped to competency evidence">
|
| 218 |
+
<Box sx={{ display: 'flex', gap: 1, flexWrap: 'wrap', mb: 1.5 }}>
|
| 219 |
+
<Chip size="small" label={`objective ${sim.debrief.objective_start} → ${sim.debrief.objective_end} (${sim.debrief.objective_delta >= 0 ? '+' : ''}${sim.debrief.objective_delta})`} sx={{ bgcolor: '#232a31', fontFamily: 'monospace' }} />
|
| 220 |
+
<Chip size="small" label={`${sim.debrief.incidents} incident(s)`} sx={{ bgcolor: sim.debrief.incidents ? '#5d1414' : '#1b5e20', color: sim.debrief.incidents ? '#ffcdd2' : '#a5d6a7' }} />
|
| 221 |
+
<Chip size="small" label={sim.debrief.balanced_finish ? 'balanced finish' : 'unbalanced finish'} sx={{ bgcolor: sim.debrief.balanced_finish ? '#1b5e20' : '#e65100', color: '#e6e9ec' }} />
|
| 222 |
+
</Box>
|
| 223 |
+
{sim.debrief.competency_mappings.map((m) => (
|
| 224 |
+
<Alert
|
| 225 |
+
key={m.competency_id}
|
| 226 |
+
severity={m.verdict === 'evidenced' ? 'success' : 'warning'}
|
| 227 |
+
variant="outlined"
|
| 228 |
+
sx={{ mb: 1, py: 0.25 }}
|
| 229 |
+
>
|
| 230 |
+
<b>{m.competency_id}</b> — {m.reason}
|
| 231 |
+
{m.quotes.map((q, i) => (
|
| 232 |
+
<Typography key={i} variant="caption" sx={{ display: 'block', fontStyle: 'italic', color: '#aab4be' }}>
|
| 233 |
+
“{q}”
|
| 234 |
+
</Typography>
|
| 235 |
+
))}
|
| 236 |
+
</Alert>
|
| 237 |
+
))}
|
| 238 |
+
<Button size="small" onClick={start}>
|
| 239 |
+
Run again (new seed)
|
| 240 |
+
</Button>
|
| 241 |
+
</SectionCard>
|
| 242 |
+
) : null}
|
| 243 |
+
</>
|
| 244 |
+
) : null}
|
| 245 |
+
</Box>
|
| 246 |
+
);
|
| 247 |
+
}
|
src/pages/hillclimb/StarView.tsx
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** STAR Story Box: paste a raw story -> diagnosed S/T/A/R with claim-vs-evidence flags. */
|
| 2 |
+
|
| 3 |
+
import { useState } from 'react';
|
| 4 |
+
import Alert from '@mui/material/Alert';
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Button from '@mui/material/Button';
|
| 7 |
+
import Chip from '@mui/material/Chip';
|
| 8 |
+
import FormControlLabel from '@mui/material/FormControlLabel';
|
| 9 |
+
import Switch from '@mui/material/Switch';
|
| 10 |
+
import TextField from '@mui/material/TextField';
|
| 11 |
+
import Typography from '@mui/material/Typography';
|
| 12 |
+
import { ErrorNote, SectionCard } from '../../components/labeleval/shared';
|
| 13 |
+
import { PassChip, ScoreChip } from '../../components/hillclimb/shared';
|
| 14 |
+
import { diagnoseStar } from '../../services/hillclimb';
|
| 15 |
+
import type { StarDiagnosis } from '../../types/hillclimb';
|
| 16 |
+
|
| 17 |
+
const COMPONENT_COLORS: Record<string, string> = {
|
| 18 |
+
S: '#4fc3f7',
|
| 19 |
+
T: '#9575cd',
|
| 20 |
+
A: '#ffb74d',
|
| 21 |
+
R: '#66bb6a',
|
| 22 |
+
};
|
| 23 |
+
|
| 24 |
+
export default function StarView() {
|
| 25 |
+
const [text, setText] = useState('');
|
| 26 |
+
const [saveEvidence, setSaveEvidence] = useState(true);
|
| 27 |
+
const [diagnosis, setDiagnosis] = useState<StarDiagnosis | null>(null);
|
| 28 |
+
const [busy, setBusy] = useState(false);
|
| 29 |
+
const [error, setError] = useState<string | null>(null);
|
| 30 |
+
|
| 31 |
+
const run = () => {
|
| 32 |
+
if (!text.trim()) return;
|
| 33 |
+
setBusy(true);
|
| 34 |
+
setError(null);
|
| 35 |
+
diagnoseStar(text, saveEvidence)
|
| 36 |
+
.then(setDiagnosis)
|
| 37 |
+
.catch((e: Error) => setError(e.message))
|
| 38 |
+
.finally(() => setBusy(false));
|
| 39 |
+
};
|
| 40 |
+
|
| 41 |
+
return (
|
| 42 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 43 |
+
<SectionCard
|
| 44 |
+
title="Your story, unstructured"
|
| 45 |
+
help="Paste a real experience in any shape. The diagnoser (not a rewriter) segments it into Situation/Task/Action/Result, runs per-component checks, flags unquantified claims vs measurable evidence, and maps the story to the Phase-3 leadership competencies it actually evidences."
|
| 46 |
+
>
|
| 47 |
+
{error ? <ErrorNote error={error} /> : null}
|
| 48 |
+
<TextField
|
| 49 |
+
multiline
|
| 50 |
+
minRows={6}
|
| 51 |
+
fullWidth
|
| 52 |
+
placeholder="e.g. 'When I joined, the team of 12 was missing every release date. I was asked to… I decided to… As a result, deploy failures dropped from 14% to 3%…'"
|
| 53 |
+
value={text}
|
| 54 |
+
onChange={(e) => setText(e.target.value)}
|
| 55 |
+
disabled={busy}
|
| 56 |
+
/>
|
| 57 |
+
<Box sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', mt: 1.5 }}>
|
| 58 |
+
<FormControlLabel
|
| 59 |
+
control={<Switch size="small" checked={saveEvidence} onChange={(e) => setSaveEvidence(e.target.checked)} />}
|
| 60 |
+
label={
|
| 61 |
+
<Typography variant="caption" sx={{ color: '#8a949e' }}>
|
| 62 |
+
Save to evidence library on diagnose
|
| 63 |
+
</Typography>
|
| 64 |
+
}
|
| 65 |
+
/>
|
| 66 |
+
<Button variant="contained" onClick={run} disabled={busy || !text.trim()}>
|
| 67 |
+
{busy ? 'Diagnosing…' : 'Diagnose story'}
|
| 68 |
+
</Button>
|
| 69 |
+
</Box>
|
| 70 |
+
</SectionCard>
|
| 71 |
+
|
| 72 |
+
{diagnosis ? (
|
| 73 |
+
<>
|
| 74 |
+
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1, flexWrap: 'wrap' }}>
|
| 75 |
+
<ScoreChip score={diagnosis.overall_score} label="Story" />
|
| 76 |
+
{diagnosis.checks.map((c) => (
|
| 77 |
+
<PassChip key={c.check} passed={c.passed} label={c.label} />
|
| 78 |
+
))}
|
| 79 |
+
{diagnosis.evidence_id ? (
|
| 80 |
+
<Chip size="small" label="saved to evidence" sx={{ bgcolor: '#00695c', color: '#80cbc4', fontSize: 11 }} />
|
| 81 |
+
) : null}
|
| 82 |
+
</Box>
|
| 83 |
+
|
| 84 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap' }}>
|
| 85 |
+
{diagnosis.components.map((comp) => (
|
| 86 |
+
<SectionCard
|
| 87 |
+
key={comp.component}
|
| 88 |
+
title={
|
| 89 |
+
<Box component="span" sx={{ color: COMPONENT_COLORS[comp.component] }}>
|
| 90 |
+
{comp.component} — {comp.label}
|
| 91 |
+
</Box>
|
| 92 |
+
}
|
| 93 |
+
sx={{ flex: '1 1 260px', borderTop: `2px solid ${COMPONENT_COLORS[comp.component]}` }}
|
| 94 |
+
>
|
| 95 |
+
{comp.present ? (
|
| 96 |
+
comp.sentences.map((s, i) => (
|
| 97 |
+
<Typography key={i} variant="body2" sx={{ color: '#cfd8dc', mb: 0.5 }}>
|
| 98 |
+
{s}
|
| 99 |
+
</Typography>
|
| 100 |
+
))
|
| 101 |
+
) : (
|
| 102 |
+
<Typography variant="body2" sx={{ color: '#ef9a9a' }}>
|
| 103 |
+
{comp.issues[0] ?? 'Not detected.'}
|
| 104 |
+
</Typography>
|
| 105 |
+
)}
|
| 106 |
+
</SectionCard>
|
| 107 |
+
))}
|
| 108 |
+
</Box>
|
| 109 |
+
|
| 110 |
+
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap' }}>
|
| 111 |
+
<SectionCard
|
| 112 |
+
title="Claim vs evidence"
|
| 113 |
+
help="Unquantified claims ('improved performance') are flagged; claims backed by numbers / before-after / named mechanisms count as measurable evidence. Unquantified claims cap the story score at 3."
|
| 114 |
+
sx={{ flex: '2 1 380px' }}
|
| 115 |
+
>
|
| 116 |
+
{diagnosis.claim_flags.length === 0 ? (
|
| 117 |
+
<Typography variant="body2" sx={{ color: '#8a949e' }}>
|
| 118 |
+
No outcome claims detected — a strong story should claim a measurable result.
|
| 119 |
+
</Typography>
|
| 120 |
+
) : (
|
| 121 |
+
diagnosis.claim_flags.map((f, i) => (
|
| 122 |
+
<Alert
|
| 123 |
+
key={i}
|
| 124 |
+
severity={f.kind === 'measurable_evidence' ? 'success' : 'warning'}
|
| 125 |
+
variant="outlined"
|
| 126 |
+
sx={{ mb: 1, py: 0.25 }}
|
| 127 |
+
>
|
| 128 |
+
<Typography variant="caption" sx={{ display: 'block', fontStyle: 'italic' }}>
|
| 129 |
+
“{f.sentence}”
|
| 130 |
+
</Typography>
|
| 131 |
+
<Typography variant="caption">{f.detail}</Typography>
|
| 132 |
+
</Alert>
|
| 133 |
+
))
|
| 134 |
+
)}
|
| 135 |
+
</SectionCard>
|
| 136 |
+
|
| 137 |
+
<SectionCard
|
| 138 |
+
title="Competencies evidenced"
|
| 139 |
+
help="Phase-3 leadership competencies this story can serve as evidence for, with the matched cues."
|
| 140 |
+
sx={{ flex: '1 1 280px' }}
|
| 141 |
+
>
|
| 142 |
+
{diagnosis.competencies.length === 0 ? (
|
| 143 |
+
<Typography variant="body2" sx={{ color: '#8a949e' }}>
|
| 144 |
+
None detected yet — anchor the story to a leadership behavior.
|
| 145 |
+
</Typography>
|
| 146 |
+
) : (
|
| 147 |
+
diagnosis.competencies.map((c) => (
|
| 148 |
+
<Box key={c.competency_id} sx={{ mb: 1 }}>
|
| 149 |
+
<Chip
|
| 150 |
+
size="small"
|
| 151 |
+
label={c.competency_id}
|
| 152 |
+
sx={{ bgcolor: '#232a31', fontFamily: 'monospace', fontSize: 11 }}
|
| 153 |
+
/>
|
| 154 |
+
<Typography variant="caption" sx={{ color: '#8a949e', display: 'block' }}>
|
| 155 |
+
{c.reason}
|
| 156 |
+
</Typography>
|
| 157 |
+
</Box>
|
| 158 |
+
))
|
| 159 |
+
)}
|
| 160 |
+
{diagnosis.coaching.length > 0 ? (
|
| 161 |
+
<>
|
| 162 |
+
<Typography variant="caption" sx={{ color: '#4fc3f7', fontWeight: 700, display: 'block', mt: 1 }}>
|
| 163 |
+
STRENGTHEN
|
| 164 |
+
</Typography>
|
| 165 |
+
{diagnosis.coaching.map((c, i) => (
|
| 166 |
+
<Typography key={i} variant="caption" sx={{ color: '#aab4be', display: 'block', mb: 0.5 }}>
|
| 167 |
+
• {c}
|
| 168 |
+
</Typography>
|
| 169 |
+
))}
|
| 170 |
+
</>
|
| 171 |
+
) : null}
|
| 172 |
+
</SectionCard>
|
| 173 |
+
</Box>
|
| 174 |
+
</>
|
| 175 |
+
) : null}
|
| 176 |
+
</Box>
|
| 177 |
+
);
|
| 178 |
+
}
|
src/services/hillclimb.ts
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Typed fetch wrappers for the Hill Climbing EM API (/api/hillclimb). */
|
| 2 |
+
|
| 3 |
+
import type {
|
| 4 |
+
Blueprint,
|
| 5 |
+
DesignChallenge,
|
| 6 |
+
DesignComponent,
|
| 7 |
+
DesignEdge,
|
| 8 |
+
DesignGrade,
|
| 9 |
+
DiagnosticSession,
|
| 10 |
+
EvidenceListResponse,
|
| 11 |
+
Exercise,
|
| 12 |
+
GraphResponse,
|
| 13 |
+
InterviewSession,
|
| 14 |
+
Journey,
|
| 15 |
+
NextBestAction,
|
| 16 |
+
ReadinessResponse,
|
| 17 |
+
SimulationCatalog,
|
| 18 |
+
SimulationState,
|
| 19 |
+
StarDiagnosis,
|
| 20 |
+
SubmitExerciseResponse,
|
| 21 |
+
UserProfile,
|
| 22 |
+
} from '../types/hillclimb';
|
| 23 |
+
|
| 24 |
+
const BASE = '/api/hillclimb';
|
| 25 |
+
const JSON_HEADERS = { 'Content-Type': 'application/json' };
|
| 26 |
+
|
| 27 |
+
async function handle<T>(res: Response): Promise<T> {
|
| 28 |
+
if (!res.ok) {
|
| 29 |
+
const body = await res.text().catch(() => '');
|
| 30 |
+
throw new Error(`API ${res.status}: ${body || res.statusText}`);
|
| 31 |
+
}
|
| 32 |
+
return res.json() as Promise<T>;
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
function get<T>(url: string): Promise<T> {
|
| 36 |
+
return fetch(`${BASE}${url}`).then((res) => handle<T>(res));
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
function post<T>(url: string, body?: unknown): Promise<T> {
|
| 40 |
+
return fetch(`${BASE}${url}`, {
|
| 41 |
+
method: 'POST',
|
| 42 |
+
headers: JSON_HEADERS,
|
| 43 |
+
body: body === undefined ? undefined : JSON.stringify(body),
|
| 44 |
+
}).then((res) => handle<T>(res));
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
function put<T>(url: string, body: unknown): Promise<T> {
|
| 48 |
+
return fetch(`${BASE}${url}`, {
|
| 49 |
+
method: 'PUT',
|
| 50 |
+
headers: JSON_HEADERS,
|
| 51 |
+
body: JSON.stringify(body),
|
| 52 |
+
}).then((res) => handle<T>(res));
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
// ------------------------------------------------------------------- profile
|
| 56 |
+
|
| 57 |
+
export const getProfile = () => get<UserProfile>('/profile');
|
| 58 |
+
export const saveProfile = (p: UserProfile) => put<UserProfile>('/profile', p);
|
| 59 |
+
|
| 60 |
+
// ----------------------------------------------------------------- blueprint
|
| 61 |
+
|
| 62 |
+
export const getBlueprint = () => get<Blueprint>('/blueprint');
|
| 63 |
+
export const getGraph = () => get<GraphResponse>('/graph');
|
| 64 |
+
|
| 65 |
+
// ---------------------------------------------------------------- diagnostic
|
| 66 |
+
|
| 67 |
+
export const startDiagnostic = (seed = 11) =>
|
| 68 |
+
post<DiagnosticSession>('/diagnostic/start', { seed });
|
| 69 |
+
export const answerDiagnostic = (diagnosticId: string, answer: string) =>
|
| 70 |
+
post<DiagnosticSession>(`/diagnostic/${encodeURIComponent(diagnosticId)}/answer`, { answer });
|
| 71 |
+
|
| 72 |
+
// ----------------------------------------------------------------- exercises
|
| 73 |
+
|
| 74 |
+
export const generateExercise = (competencyId: string, difficulty = 2, seed?: number) =>
|
| 75 |
+
post<Exercise>('/exercise/generate', { competency_id: competencyId, difficulty, seed });
|
| 76 |
+
export const submitExercise = (exerciseId: string, answer: string, asAssessment = false) =>
|
| 77 |
+
post<SubmitExerciseResponse>('/exercise/submit', {
|
| 78 |
+
exercise_id: exerciseId,
|
| 79 |
+
answer,
|
| 80 |
+
as_assessment: asAssessment,
|
| 81 |
+
});
|
| 82 |
+
|
| 83 |
+
// ---------------------------------------------------------------------- STAR
|
| 84 |
+
|
| 85 |
+
export const diagnoseStar = (text: string, saveEvidence: boolean) =>
|
| 86 |
+
post<StarDiagnosis>('/star/diagnose', { text, save_evidence: saveEvidence });
|
| 87 |
+
|
| 88 |
+
// ----------------------------------------------------------------- design lab
|
| 89 |
+
|
| 90 |
+
export const getDesignChallenges = () =>
|
| 91 |
+
get<{ challenges: DesignChallenge[]; component_types: string[] }>('/design/challenges');
|
| 92 |
+
export const submitDesign = (
|
| 93 |
+
challengeId: string,
|
| 94 |
+
components: DesignComponent[],
|
| 95 |
+
edges: DesignEdge[],
|
| 96 |
+
rationales: Record<string, string>
|
| 97 |
+
) =>
|
| 98 |
+
post<DesignGrade>('/design/submit', {
|
| 99 |
+
challenge_id: challengeId,
|
| 100 |
+
components,
|
| 101 |
+
edges,
|
| 102 |
+
rationales,
|
| 103 |
+
});
|
| 104 |
+
|
| 105 |
+
// ----------------------------------------------------------------- simulation
|
| 106 |
+
|
| 107 |
+
export const getSimulationCatalog = () => get<SimulationCatalog>('/simulation/catalog');
|
| 108 |
+
export const startSimulation = (seed = 42, maxTurns = 8) =>
|
| 109 |
+
post<SimulationState>('/simulation/start', { seed, max_turns: maxTurns });
|
| 110 |
+
export const stepSimulation = (
|
| 111 |
+
simId: string,
|
| 112 |
+
hypothesis: string,
|
| 113 |
+
interventionId: string,
|
| 114 |
+
revertPrevious = false
|
| 115 |
+
) =>
|
| 116 |
+
post<SimulationState>(`/simulation/${encodeURIComponent(simId)}/step`, {
|
| 117 |
+
hypothesis,
|
| 118 |
+
intervention_id: interventionId,
|
| 119 |
+
revert_previous: revertPrevious,
|
| 120 |
+
});
|
| 121 |
+
|
| 122 |
+
// ------------------------------------------------------------------ interview
|
| 123 |
+
|
| 124 |
+
export const startInterview = (mode: string, seed?: number) =>
|
| 125 |
+
post<InterviewSession>('/interview/start', seed === undefined ? { mode } : { mode, seed });
|
| 126 |
+
export const answerInterview = (sessionId: string, answer: string) =>
|
| 127 |
+
post<InterviewSession>(`/interview/${encodeURIComponent(sessionId)}/answer`, { answer });
|
| 128 |
+
export const endInterview = (sessionId: string) =>
|
| 129 |
+
post<InterviewSession>(`/interview/${encodeURIComponent(sessionId)}/end`);
|
| 130 |
+
|
| 131 |
+
// ----------------------------------------------------- evidence / readiness
|
| 132 |
+
|
| 133 |
+
export const getEvidence = (competencyId?: string) =>
|
| 134 |
+
get<EvidenceListResponse>(
|
| 135 |
+
competencyId ? `/evidence?competency_id=${encodeURIComponent(competencyId)}` : '/evidence'
|
| 136 |
+
);
|
| 137 |
+
export const getReadiness = () => get<ReadinessResponse>('/readiness');
|
| 138 |
+
export const getNextAction = () => get<NextBestAction>('/next-action');
|
| 139 |
+
export const getJourney = () => get<Journey>('/journey');
|
src/types/hillclimb.ts
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Types for the Hill Climbing EM API (/api/hillclimb). */
|
| 2 |
+
|
| 3 |
+
export type Dimension = 'Knowledge' | 'Technical Reasoning' | 'Leadership' | 'Execution';
|
| 4 |
+
|
| 5 |
+
export type ReadinessStateId =
|
| 6 |
+
| 'NOT_STARTED'
|
| 7 |
+
| 'LEARNING'
|
| 8 |
+
| 'PRACTICING'
|
| 9 |
+
| 'NEEDS_REVIEW'
|
| 10 |
+
| 'COMPETENT'
|
| 11 |
+
| 'STRONG'
|
| 12 |
+
| 'INTERVIEW_READY';
|
| 13 |
+
|
| 14 |
+
export type JourneyStateId =
|
| 15 |
+
| 'NOT_STARTED'
|
| 16 |
+
| 'DIAGNOSTIC'
|
| 17 |
+
| 'LEARNING'
|
| 18 |
+
| 'PRACTICE'
|
| 19 |
+
| 'ASSESSMENT'
|
| 20 |
+
| 'REMEDIATION'
|
| 21 |
+
| 'REASSESS';
|
| 22 |
+
|
| 23 |
+
export interface UserProfile {
|
| 24 |
+
user_id: string;
|
| 25 |
+
name: string;
|
| 26 |
+
target_role: string;
|
| 27 |
+
experience_years: number;
|
| 28 |
+
focus_note: string;
|
| 29 |
+
created_at: string;
|
| 30 |
+
updated_at: string;
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
// ------------------------------------------------------------------ blueprint
|
| 34 |
+
|
| 35 |
+
export interface Competency {
|
| 36 |
+
id: string;
|
| 37 |
+
name: string;
|
| 38 |
+
phase: number;
|
| 39 |
+
dimension: Dimension;
|
| 40 |
+
description: string;
|
| 41 |
+
prerequisites: string[];
|
| 42 |
+
topics: string[];
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
export interface PhaseSpec {
|
| 46 |
+
phase: number;
|
| 47 |
+
title: string;
|
| 48 |
+
objective: string;
|
| 49 |
+
topics: string[];
|
| 50 |
+
skills: string[];
|
| 51 |
+
exercises: string[];
|
| 52 |
+
assessments: string[];
|
| 53 |
+
completion_criteria: string[];
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
export interface Blueprint {
|
| 57 |
+
version: number;
|
| 58 |
+
source: string;
|
| 59 |
+
note: string;
|
| 60 |
+
phases: PhaseSpec[];
|
| 61 |
+
competencies: Competency[];
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
export interface GraphResponse {
|
| 65 |
+
nodes: Competency[];
|
| 66 |
+
edges: { source: string; target: string }[];
|
| 67 |
+
problems: string[];
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
// ----------------------------------------------------------------- evaluation
|
| 71 |
+
|
| 72 |
+
export interface EvaluationResult {
|
| 73 |
+
competency: string;
|
| 74 |
+
score: number;
|
| 75 |
+
confidence: number;
|
| 76 |
+
evidence: string[];
|
| 77 |
+
strengths: string[];
|
| 78 |
+
weaknesses: string[];
|
| 79 |
+
missing_evidence: string[];
|
| 80 |
+
misconceptions: string[];
|
| 81 |
+
recommended_action: string;
|
| 82 |
+
follow_up_question: string;
|
| 83 |
+
evaluator: string;
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
export interface RubricItem {
|
| 87 |
+
criterion: string;
|
| 88 |
+
check: string;
|
| 89 |
+
keywords: string[];
|
| 90 |
+
weight: number;
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
export interface LinkedTool {
|
| 94 |
+
label: string;
|
| 95 |
+
page: string;
|
| 96 |
+
api: string;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
export interface Exercise {
|
| 100 |
+
exercise_id: string;
|
| 101 |
+
competency_id: string;
|
| 102 |
+
difficulty: number;
|
| 103 |
+
prerequisites: string[];
|
| 104 |
+
scenario: string;
|
| 105 |
+
expected_reasoning: string[];
|
| 106 |
+
evaluation_rubric: RubricItem[];
|
| 107 |
+
common_failure_modes: string[];
|
| 108 |
+
follow_up_questions: string[];
|
| 109 |
+
template_id: string;
|
| 110 |
+
seed: number;
|
| 111 |
+
family: string;
|
| 112 |
+
linked_tool: LinkedTool | null;
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
export interface CompetencyReadiness {
|
| 116 |
+
competency_id: string;
|
| 117 |
+
knowledge_score: number;
|
| 118 |
+
application_score: number;
|
| 119 |
+
evidence_score: number;
|
| 120 |
+
readiness_state: ReadinessStateId;
|
| 121 |
+
evidence_ids: string[];
|
| 122 |
+
last_updated: string;
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
export interface SubmitExerciseResponse {
|
| 126 |
+
evaluation: EvaluationResult;
|
| 127 |
+
coaching: string;
|
| 128 |
+
attempt_id: string;
|
| 129 |
+
evidence_id: string;
|
| 130 |
+
linked_tool: LinkedTool | null;
|
| 131 |
+
readiness_for_competency: CompetencyReadiness | null;
|
| 132 |
+
journey: Journey | null;
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
// ------------------------------------------------------------------ diagnostic
|
| 136 |
+
|
| 137 |
+
export interface DiagnosticQuestion {
|
| 138 |
+
exercise_id: string;
|
| 139 |
+
competency_id: string;
|
| 140 |
+
scenario: string;
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
export interface DiagnosticSession {
|
| 144 |
+
diagnostic_id: string;
|
| 145 |
+
user_id: string;
|
| 146 |
+
seed: number;
|
| 147 |
+
total_questions: number;
|
| 148 |
+
answered: number;
|
| 149 |
+
status: 'active' | 'complete';
|
| 150 |
+
current_exercise_id: string | null;
|
| 151 |
+
current_competency: string | null;
|
| 152 |
+
asked: string[];
|
| 153 |
+
results: { competency_id: string; score: number; evaluation: EvaluationResult }[];
|
| 154 |
+
current_question?: DiagnosticQuestion;
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
// ------------------------------------------------------------------------ STAR
|
| 158 |
+
|
| 159 |
+
export interface StarComponent {
|
| 160 |
+
component: 'S' | 'T' | 'A' | 'R';
|
| 161 |
+
label: string;
|
| 162 |
+
sentences: string[];
|
| 163 |
+
present: boolean;
|
| 164 |
+
issues: string[];
|
| 165 |
+
strengths: string[];
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
export interface ClaimFlag {
|
| 169 |
+
sentence: string;
|
| 170 |
+
kind: 'unquantified_claim' | 'measurable_evidence';
|
| 171 |
+
detail: string;
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
export interface StarCheck {
|
| 175 |
+
check: string;
|
| 176 |
+
label: string;
|
| 177 |
+
passed: boolean;
|
| 178 |
+
detail: string;
|
| 179 |
+
evidence: string | null;
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
export interface StarDiagnosis {
|
| 183 |
+
components: StarComponent[];
|
| 184 |
+
checks: StarCheck[];
|
| 185 |
+
claim_flags: ClaimFlag[];
|
| 186 |
+
competencies: { competency_id: string; matched_cues: string[]; reason: string }[];
|
| 187 |
+
overall_score: number;
|
| 188 |
+
coaching: string[];
|
| 189 |
+
evidence_id: string | null;
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
// ------------------------------------------------------------------ design lab
|
| 193 |
+
|
| 194 |
+
export interface DesignChallenge {
|
| 195 |
+
challenge_id: string;
|
| 196 |
+
title: string;
|
| 197 |
+
brief: string;
|
| 198 |
+
required_stages: string[][];
|
| 199 |
+
key_decisions: string[];
|
| 200 |
+
competency_ids: string[];
|
| 201 |
+
requires_feedback_loop: boolean;
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
export interface DesignComponent {
|
| 205 |
+
id: string;
|
| 206 |
+
type: string;
|
| 207 |
+
name: string;
|
| 208 |
+
note: string;
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
export interface DesignEdge {
|
| 212 |
+
source: string;
|
| 213 |
+
target: string;
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
export interface StructuralChecks {
|
| 217 |
+
missing_stages: string[];
|
| 218 |
+
orphan_components: string[];
|
| 219 |
+
feedback_loop_closed: boolean;
|
| 220 |
+
feedback_loop_required: boolean;
|
| 221 |
+
single_points_of_failure: string[];
|
| 222 |
+
capacity_math_found: boolean;
|
| 223 |
+
capacity_quotes: string[];
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
export interface DimensionGrade {
|
| 227 |
+
dimension: string;
|
| 228 |
+
score: number;
|
| 229 |
+
gaps: string[];
|
| 230 |
+
evidence: string[];
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
export interface DesignGrade {
|
| 234 |
+
grade_id: string;
|
| 235 |
+
challenge_id: string;
|
| 236 |
+
structural: StructuralChecks;
|
| 237 |
+
dimension_grades: DimensionGrade[];
|
| 238 |
+
overall_score: number;
|
| 239 |
+
gaps: string[];
|
| 240 |
+
evidence_id: string | null;
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
// ------------------------------------------------------------------ simulation
|
| 244 |
+
|
| 245 |
+
export interface InterventionSpec {
|
| 246 |
+
id: string;
|
| 247 |
+
label: string;
|
| 248 |
+
description: string;
|
| 249 |
+
immediate: Record<string, number>;
|
| 250 |
+
delayed: { delay: number; effects: Record<string, number>; note?: string }[];
|
| 251 |
+
second_order: string;
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
export interface SimulationCatalog {
|
| 255 |
+
scenarios: { scenario_id: string; title: string; narrative: string }[];
|
| 256 |
+
interventions: InterventionSpec[];
|
| 257 |
+
metrics: string[];
|
| 258 |
+
inverted_metrics: string[];
|
| 259 |
+
safety_floor: number;
|
| 260 |
+
morale_floor: number;
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
export interface SimTurnRecord {
|
| 264 |
+
turn: number;
|
| 265 |
+
hypothesis: string;
|
| 266 |
+
hypothesis_assessment: {
|
| 267 |
+
mentions_metrics: string[];
|
| 268 |
+
targets_intervention_effects: boolean;
|
| 269 |
+
directional: boolean;
|
| 270 |
+
falsifiable: boolean;
|
| 271 |
+
quality: string;
|
| 272 |
+
};
|
| 273 |
+
intervention_id: string;
|
| 274 |
+
reverted_previous: boolean;
|
| 275 |
+
applied_effects: Record<string, number>;
|
| 276 |
+
delayed_landed: string[];
|
| 277 |
+
events: string[];
|
| 278 |
+
metrics_after: Record<string, number>;
|
| 279 |
+
objective_after: number;
|
| 280 |
+
objective_delta: number;
|
| 281 |
+
verdict: string;
|
| 282 |
+
timestamp: string;
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
export interface SimCompetencyMapping {
|
| 286 |
+
competency_id: string;
|
| 287 |
+
verdict: 'evidenced' | 'gap';
|
| 288 |
+
reason: string;
|
| 289 |
+
quotes: string[];
|
| 290 |
+
evidence_id?: string;
|
| 291 |
+
}
|
| 292 |
+
|
| 293 |
+
export interface SimDebrief {
|
| 294 |
+
objective_start: number;
|
| 295 |
+
objective_end: number;
|
| 296 |
+
objective_delta: number;
|
| 297 |
+
incidents: number;
|
| 298 |
+
turns: number;
|
| 299 |
+
balanced_finish: boolean;
|
| 300 |
+
competency_mappings: SimCompetencyMapping[];
|
| 301 |
+
}
|
| 302 |
+
|
| 303 |
+
export interface SimulationState {
|
| 304 |
+
sim_id: string;
|
| 305 |
+
user_id: string;
|
| 306 |
+
scenario_id: string;
|
| 307 |
+
seed: number;
|
| 308 |
+
max_turns: number;
|
| 309 |
+
turn: number;
|
| 310 |
+
status: 'active' | 'complete';
|
| 311 |
+
metrics: Record<string, number>;
|
| 312 |
+
pending: { due_turn: number; effects: Record<string, number>; note: string }[];
|
| 313 |
+
history: SimTurnRecord[];
|
| 314 |
+
events: { turn: number; type: string; detail: string }[];
|
| 315 |
+
objective_history: number[];
|
| 316 |
+
debrief: SimDebrief | null;
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
// -------------------------------------------------------------------- interview
|
| 320 |
+
|
| 321 |
+
export interface InterviewTurn {
|
| 322 |
+
index: number;
|
| 323 |
+
question: string;
|
| 324 |
+
question_type: 'opening' | 'probe' | 'depth_probe' | 'escalate' | 'advance';
|
| 325 |
+
competency_id: string;
|
| 326 |
+
exercise_id: string;
|
| 327 |
+
difficulty: number;
|
| 328 |
+
answer: string | null;
|
| 329 |
+
evaluation: EvaluationResult | null;
|
| 330 |
+
timestamp: string;
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
export interface InterviewSession {
|
| 334 |
+
session_id: string;
|
| 335 |
+
user_id: string;
|
| 336 |
+
mode: 'technical' | 'management' | 'hybrid';
|
| 337 |
+
seed: number;
|
| 338 |
+
status: 'active' | 'complete';
|
| 339 |
+
turns: InterviewTurn[];
|
| 340 |
+
asked_competencies: string[];
|
| 341 |
+
probe_count_on_current: number;
|
| 342 |
+
evidence_id: string | null;
|
| 343 |
+
}
|
| 344 |
+
|
| 345 |
+
// -------------------------------------------------------- evidence / readiness
|
| 346 |
+
|
| 347 |
+
export interface EvidenceItem {
|
| 348 |
+
evidence_id: string;
|
| 349 |
+
user_id: string;
|
| 350 |
+
competency_ids: string[];
|
| 351 |
+
artifact_type: string;
|
| 352 |
+
source: string;
|
| 353 |
+
summary: string;
|
| 354 |
+
quotes: string[];
|
| 355 |
+
score: number;
|
| 356 |
+
confidence: number;
|
| 357 |
+
timestamp: string;
|
| 358 |
+
payload: Record<string, unknown>;
|
| 359 |
+
}
|
| 360 |
+
|
| 361 |
+
export interface EvidenceListResponse {
|
| 362 |
+
evidence: EvidenceItem[];
|
| 363 |
+
count: number;
|
| 364 |
+
}
|
| 365 |
+
|
| 366 |
+
export interface ReadinessRow extends CompetencyReadiness {
|
| 367 |
+
name: string;
|
| 368 |
+
phase: number;
|
| 369 |
+
dimension: Dimension;
|
| 370 |
+
prerequisites: string[];
|
| 371 |
+
}
|
| 372 |
+
|
| 373 |
+
export interface DimensionSummary {
|
| 374 |
+
dimension: string;
|
| 375 |
+
avg_score: number;
|
| 376 |
+
coverage: number;
|
| 377 |
+
competent: number;
|
| 378 |
+
total: number;
|
| 379 |
+
}
|
| 380 |
+
|
| 381 |
+
export interface Bottleneck {
|
| 382 |
+
competency_id: string;
|
| 383 |
+
name: string;
|
| 384 |
+
phase: number;
|
| 385 |
+
dimension: Dimension;
|
| 386 |
+
readiness: CompetencyReadiness;
|
| 387 |
+
blocked_competencies: string[];
|
| 388 |
+
blocked_count: number;
|
| 389 |
+
explanation: string;
|
| 390 |
+
}
|
| 391 |
+
|
| 392 |
+
export interface ReadinessResponse {
|
| 393 |
+
matrix: ReadinessRow[];
|
| 394 |
+
dimensions: DimensionSummary[];
|
| 395 |
+
bottleneck: Bottleneck | null;
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
export interface NextBestAction {
|
| 399 |
+
bottleneck: Bottleneck | null;
|
| 400 |
+
explanation: string;
|
| 401 |
+
concept: {
|
| 402 |
+
competency_id: string;
|
| 403 |
+
name: string;
|
| 404 |
+
study: string[];
|
| 405 |
+
description: string;
|
| 406 |
+
phase_objective: string;
|
| 407 |
+
};
|
| 408 |
+
exercise: Exercise;
|
| 409 |
+
assessment: {
|
| 410 |
+
competency_id: string;
|
| 411 |
+
kind: string;
|
| 412 |
+
description: string;
|
| 413 |
+
};
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
export interface Journey {
|
| 417 |
+
user_id: string;
|
| 418 |
+
state: JourneyStateId;
|
| 419 |
+
current_phase: number;
|
| 420 |
+
current_competency: string | null;
|
| 421 |
+
remediation_target: string | null;
|
| 422 |
+
history: Record<string, unknown>[];
|
| 423 |
+
updated_at: string;
|
| 424 |
+
}
|
tests/test_hillclimb/__init__.py
ADDED
|
File without changes
|
tests/test_hillclimb/conftest.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pytest
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb.models import reset_store
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@pytest.fixture(autouse=True)
|
| 7 |
+
def isolated_store(tmp_path, monkeypatch):
|
| 8 |
+
"""Every test runs against a fresh store and with the LLM disabled, so the
|
| 9 |
+
deterministic rule-based paths are what is under test."""
|
| 10 |
+
monkeypatch.setenv("HILLCLIMB_DISABLE_LLM", "1")
|
| 11 |
+
store = reset_store(tmp_path / "hillclimb")
|
| 12 |
+
yield store
|
| 13 |
+
reset_store(tmp_path / "hillclimb-teardown")
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
# ---------------------------------------------------------- shared answer text
|
| 17 |
+
|
| 18 |
+
# Covers nearly every rubric criterion of the offline/shadow exercise family,
|
| 19 |
+
# with quantified claims and explicit tradeoffs.
|
| 20 |
+
STRONG_TECH_ANSWER = (
|
| 21 |
+
"First I would check for a metric definition mismatch: offline eval and shadow monitoring may "
|
| 22 |
+
"use a different IoU threshold or matching threshold, so the two numbers are not the same metric. "
|
| 23 |
+
"Second, distribution shift: live traffic differs from the eval set, so the population changed. "
|
| 24 |
+
"Third, sampling bias in how the eval set was built — it may not be representative. "
|
| 25 |
+
"Fourth, training-serving skew: the feature pipeline can produce a feature mismatch at serving time. "
|
| 26 |
+
"Fifth, a serving mismatch: preprocessing or quantization differences in the inference stack. "
|
| 27 |
+
"I would also test statistical significance — the shadow delta may be noise, so I would compute a "
|
| 28 |
+
"95% confidence interval given the sample size. A true regression remains possible, so I would "
|
| 29 |
+
"slice metrics by scenario, run a matched A/B experiment, and re-score identical frames in both stacks. "
|
| 30 |
+
"The tradeoff: this delays launch by 3 days, at the cost of schedule, but avoids shipping a "
|
| 31 |
+
"genuinely worse model."
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
# Long, polished-sounding, but covers no rubric concept and has no numbers.
|
| 35 |
+
VERBOSE_WAFFLE = (
|
| 36 |
+
"This is truly an interesting and important challenge, and in situations like this one it is "
|
| 37 |
+
"essential to take a holistic, thoughtful, and collaborative approach that considers the "
|
| 38 |
+
"perspectives of all involved parties. I believe the most valuable thing a leader can do is to "
|
| 39 |
+
"bring people together, encourage open communication, and ensure that everyone feels heard and "
|
| 40 |
+
"aligned around a shared vision of success. It is also important to remember that great teams "
|
| 41 |
+
"are built on trust, and trust is built through consistency, empathy, and transparency. "
|
| 42 |
+
"I would begin by gathering the relevant people in a room and facilitating a constructive "
|
| 43 |
+
"discussion about the situation, its causes, and the potential paths forward, always keeping an "
|
| 44 |
+
"open mind and remaining flexible as new information emerges. Ultimately, what matters most is "
|
| 45 |
+
"that we move forward together with confidence, clarity, and a renewed sense of purpose, and "
|
| 46 |
+
"that we continue to learn and grow as an organization through every challenge we encounter. "
|
| 47 |
+
"By staying positive, communicating openly, and supporting one another, I am confident that "
|
| 48 |
+
"we will arrive at a good outcome for the business, for the team, and for our users, and that "
|
| 49 |
+
"this challenge will make us stronger as a group going forward."
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
CONCISE_PRECISE_ANSWER = (
|
| 53 |
+
"Likely causes: a metric definition mismatch between the two pipelines, distribution shift "
|
| 54 |
+
"between eval data and live traffic, or sampling bias in the eval set. Before reacting I would "
|
| 55 |
+
"check statistical significance of the shadow delta."
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
STRONG_LEADERSHIP_ANSWER = (
|
| 59 |
+
"When our launch slipped I decided to pause the feature work. Instead of quietly absorbing the "
|
| 60 |
+
"risk, I considered delaying the launch or cutting scope, and rejected the delay because it "
|
| 61 |
+
"punished the wrong team. I held a 1:1 with the tech lead, gave direct feedback, and aligned "
|
| 62 |
+
"the stakeholders on the new plan. We agreed to measure the result weekly, and the outcome was "
|
| 63 |
+
"a 30% drop in escaped regressions within 2 months. The tradeoff: we gave up one roadmap item "
|
| 64 |
+
"at the cost of short-term velocity."
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
WEAK_ANSWER = "Honestly not sure. Probably fine either way."
|
tests/test_hillclimb/test_adaptive.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Diagnostic flow adaptivity and next-best-action shape."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb import adaptive
|
| 4 |
+
from sensorflow.hillclimb.blueprint import competency_index, load_blueprint
|
| 5 |
+
|
| 6 |
+
from .conftest import STRONG_TECH_ANSWER, WEAK_ANSWER
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def test_diagnostic_seeds_matrix_and_completes(isolated_store):
|
| 10 |
+
dx = adaptive.start_diagnostic(store=isolated_store)
|
| 11 |
+
assert dx.status == "active" and dx.current_exercise_id
|
| 12 |
+
answers = [WEAK_ANSWER, STRONG_TECH_ANSWER, WEAK_ANSWER,
|
| 13 |
+
STRONG_TECH_ANSWER, WEAK_ANSWER, WEAK_ANSWER]
|
| 14 |
+
for a in answers:
|
| 15 |
+
dx = adaptive.answer_diagnostic(dx.diagnostic_id, a, use_llm=False, store=isolated_store)
|
| 16 |
+
assert dx.status == "complete"
|
| 17 |
+
assert dx.answered == 6
|
| 18 |
+
# attempts recorded -> matrix seeded
|
| 19 |
+
attempts = isolated_store.all("attempts")
|
| 20 |
+
assert len(attempts) == 6
|
| 21 |
+
assert all(a["kind"] == "diagnostic" for a in attempts)
|
| 22 |
+
# journey advanced past diagnostic
|
| 23 |
+
j = adaptive.get_journey(store=isolated_store)
|
| 24 |
+
assert j.state.value == "LEARNING"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def test_diagnostic_adapts_weak_answer_drills_into_prerequisites(isolated_store):
|
| 28 |
+
idx = competency_index(load_blueprint(isolated_store))
|
| 29 |
+
dx = adaptive.start_diagnostic(store=isolated_store)
|
| 30 |
+
first = dx.current_competency
|
| 31 |
+
dx = adaptive.answer_diagnostic(dx.diagnostic_id, WEAK_ANSWER, use_llm=False,
|
| 32 |
+
store=isolated_store)
|
| 33 |
+
second = dx.current_competency
|
| 34 |
+
prereqs = idx[first].prerequisites
|
| 35 |
+
if prereqs: # weak answer -> drill into a prerequisite
|
| 36 |
+
assert second in prereqs
|
| 37 |
+
else: # no prereqs -> continue the starter set
|
| 38 |
+
assert second != first
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def test_next_best_action_returns_exactly_one_of_each(isolated_store):
|
| 42 |
+
nba = adaptive.next_best_action(store=isolated_store)
|
| 43 |
+
assert set(nba) >= {"concept", "exercise", "assessment", "explanation", "bottleneck"}
|
| 44 |
+
assert isinstance(nba["concept"], dict) and nba["concept"]["competency_id"]
|
| 45 |
+
assert isinstance(nba["exercise"], dict) and nba["exercise"]["scenario"]
|
| 46 |
+
assert isinstance(nba["assessment"], dict) and nba["assessment"]["description"]
|
| 47 |
+
# all three target the same bottleneck competency
|
| 48 |
+
target = nba["concept"]["competency_id"]
|
| 49 |
+
assert nba["exercise"]["competency_id"] == target
|
| 50 |
+
assert nba["assessment"]["competency_id"] == target
|
tests/test_hillclimb/test_api.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""End-to-end API lifecycle through the FastAPI router (in-process TestClient)."""
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
from fastapi import FastAPI
|
| 5 |
+
from fastapi.testclient import TestClient
|
| 6 |
+
|
| 7 |
+
from sensorflow.hillclimb.api import router
|
| 8 |
+
|
| 9 |
+
from .conftest import STRONG_LEADERSHIP_ANSWER, STRONG_TECH_ANSWER, WEAK_ANSWER
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@pytest.fixture()
|
| 13 |
+
def client(isolated_store):
|
| 14 |
+
app = FastAPI()
|
| 15 |
+
app.include_router(router)
|
| 16 |
+
return TestClient(app)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_full_lifecycle(client):
|
| 20 |
+
# profile
|
| 21 |
+
p = client.get("/api/hillclimb/profile").json()
|
| 22 |
+
assert p["user_id"] == "default"
|
| 23 |
+
p["target_role"] = "EM, ML Infrastructure"
|
| 24 |
+
assert client.put("/api/hillclimb/profile", json=p).json()["target_role"] == "EM, ML Infrastructure"
|
| 25 |
+
|
| 26 |
+
# blueprint + graph
|
| 27 |
+
bp = client.get("/api/hillclimb/blueprint").json()
|
| 28 |
+
assert bp["source"] == "reconstructed-from-spec"
|
| 29 |
+
assert len(bp["phases"]) == 4
|
| 30 |
+
graph = client.get("/api/hillclimb/graph").json()
|
| 31 |
+
assert graph["problems"] == []
|
| 32 |
+
assert len(graph["nodes"]) >= 30 and graph["edges"]
|
| 33 |
+
|
| 34 |
+
# journey starts NOT_STARTED
|
| 35 |
+
assert client.get("/api/hillclimb/journey").json()["state"] == "NOT_STARTED"
|
| 36 |
+
|
| 37 |
+
# diagnostic
|
| 38 |
+
dx = client.post("/api/hillclimb/diagnostic/start", json={"seed": 11}).json()
|
| 39 |
+
assert dx["current_question"]["scenario"]
|
| 40 |
+
for _ in range(6):
|
| 41 |
+
dx = client.post(f"/api/hillclimb/diagnostic/{dx['diagnostic_id']}/answer",
|
| 42 |
+
json={"answer": STRONG_TECH_ANSWER}).json()
|
| 43 |
+
assert dx["status"] == "complete"
|
| 44 |
+
assert client.get("/api/hillclimb/journey").json()["state"] == "LEARNING"
|
| 45 |
+
|
| 46 |
+
# exercise generate + submit -> coached, evidence-backed evaluation
|
| 47 |
+
ex = client.post("/api/hillclimb/exercise/generate",
|
| 48 |
+
json={"competency_id": "p1.regression_detection", "seed": 77}).json()
|
| 49 |
+
sub = client.post("/api/hillclimb/exercise/submit",
|
| 50 |
+
json={"exercise_id": ex["exercise_id"], "answer": STRONG_TECH_ANSWER}).json()
|
| 51 |
+
assert sub["evaluation"]["score"] >= 4
|
| 52 |
+
assert sub["evaluation"]["evidence"]
|
| 53 |
+
assert sub["coaching"]
|
| 54 |
+
assert sub["linked_tool"]["api"] == "/api/rca"
|
| 55 |
+
|
| 56 |
+
# STAR
|
| 57 |
+
star = client.post("/api/hillclimb/star/diagnose", json={
|
| 58 |
+
"text": ("When I joined the team of 12 our deploys failed under a hard deadline. I was "
|
| 59 |
+
"asked to fix it. I decided to build a gate and convinced the tech lead who "
|
| 60 |
+
"disagreed. As a result failures dropped from 14% to 3% and we sustained it "
|
| 61 |
+
"with weekly monitoring since then."),
|
| 62 |
+
"save_evidence": True}).json()
|
| 63 |
+
assert star["evidence_id"]
|
| 64 |
+
assert len(star["components"]) == 4
|
| 65 |
+
|
| 66 |
+
# design lab
|
| 67 |
+
challenges = client.get("/api/hillclimb/design/challenges").json()
|
| 68 |
+
assert len(challenges["challenges"]) == 4
|
| 69 |
+
grade = client.post("/api/hillclimb/design/submit", json={
|
| 70 |
+
"challenge_id": "parallel_inference",
|
| 71 |
+
"components": [
|
| 72 |
+
{"id": "s", "type": "source", "name": "traffic", "note": "20000 qps"},
|
| 73 |
+
{"id": "i1", "type": "inference", "name": "gpu pool a"},
|
| 74 |
+
{"id": "i2", "type": "inference", "name": "gpu pool b"},
|
| 75 |
+
{"id": "st", "type": "storage", "name": "results"},
|
| 76 |
+
{"id": "m", "type": "monitoring", "name": "slo monitor"},
|
| 77 |
+
],
|
| 78 |
+
"edges": [{"source": "s", "target": "i1"}, {"source": "s", "target": "i2"},
|
| 79 |
+
{"source": "i1", "target": "st"}, {"source": "i2", "target": "st"},
|
| 80 |
+
{"source": "st", "target": "m"}],
|
| 81 |
+
"rationales": {"batching": "dynamic batching keeps p99 latency under 100 ms with 30% "
|
| 82 |
+
"headroom; the tradeoff is cost, and replicated zones give "
|
| 83 |
+
"reliability; monitoring alerts on scalability limits and "
|
| 84 |
+
"failure handling uses degraded single-model fallback"},
|
| 85 |
+
}).json()
|
| 86 |
+
assert grade["structural"]["missing_stages"] == []
|
| 87 |
+
assert grade["overall_score"] >= 2
|
| 88 |
+
|
| 89 |
+
# simulation: 3 turns
|
| 90 |
+
sim = client.post("/api/hillclimb/simulation/start", json={"seed": 42, "max_turns": 3}).json()
|
| 91 |
+
for hyp, iid in [("Monitoring will improve safety by 5", "add_monitoring"),
|
| 92 |
+
("Pausing launches raises safety by 8", "pause_launches"),
|
| 93 |
+
("1:1s will recover team morale by 5", "team_1on1s")]:
|
| 94 |
+
sim = client.post(f"/api/hillclimb/simulation/{sim['sim_id']}/step",
|
| 95 |
+
json={"hypothesis": hyp, "intervention_id": iid}).json()
|
| 96 |
+
assert sim["status"] == "complete"
|
| 97 |
+
assert sim["debrief"]["competency_mappings"]
|
| 98 |
+
catalog = client.get("/api/hillclimb/simulation/catalog").json()
|
| 99 |
+
assert catalog["interventions"] and catalog["scenarios"]
|
| 100 |
+
|
| 101 |
+
# interview: 2 questions
|
| 102 |
+
iv = client.post("/api/hillclimb/interview/start", json={"mode": "management", "seed": 5}).json()
|
| 103 |
+
iv = client.post(f"/api/hillclimb/interview/{iv['session_id']}/answer",
|
| 104 |
+
json={"answer": WEAK_ANSWER}).json()
|
| 105 |
+
assert iv["turns"][-1]["question_type"] == "probe"
|
| 106 |
+
iv = client.post(f"/api/hillclimb/interview/{iv['session_id']}/answer",
|
| 107 |
+
json={"answer": STRONG_LEADERSHIP_ANSWER}).json()
|
| 108 |
+
iv = client.post(f"/api/hillclimb/interview/{iv['session_id']}/end").json()
|
| 109 |
+
assert iv["status"] == "complete" and iv["evidence_id"]
|
| 110 |
+
|
| 111 |
+
# evidence library reflects everything above
|
| 112 |
+
ev = client.get("/api/hillclimb/evidence").json()
|
| 113 |
+
types = {e["artifact_type"] for e in ev["evidence"]}
|
| 114 |
+
assert {"star_story", "design_submission", "interview_transcript",
|
| 115 |
+
"exercise_attempt", "diagnostic"} <= types
|
| 116 |
+
|
| 117 |
+
# readiness matrix + bottleneck + next best action
|
| 118 |
+
rd = client.get("/api/hillclimb/readiness").json()
|
| 119 |
+
assert rd["matrix"] and rd["dimensions"]
|
| 120 |
+
assert any(r["knowledge_score"] > 0 for r in rd["matrix"])
|
| 121 |
+
nba = client.get("/api/hillclimb/next-action").json()
|
| 122 |
+
assert nba["concept"] and nba["exercise"] and nba["assessment"]
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def test_error_paths(client):
|
| 126 |
+
assert client.get("/api/hillclimb/exercise/nope").status_code == 404
|
| 127 |
+
assert client.post("/api/hillclimb/exercise/generate",
|
| 128 |
+
json={"competency_id": "bogus"}).status_code == 400
|
| 129 |
+
assert client.post("/api/hillclimb/star/diagnose", json={"text": " "}).status_code == 400
|
| 130 |
+
assert client.get("/api/hillclimb/simulation/nope").status_code == 404
|
| 131 |
+
assert client.post("/api/hillclimb/interview/start",
|
| 132 |
+
json={"mode": "bogus"}).status_code == 400
|
| 133 |
+
assert client.post("/api/hillclimb/journey/advance",
|
| 134 |
+
json={"event": "assessment_result", "passed": True}).status_code == 400
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def test_blueprint_edit_validation(client):
|
| 138 |
+
bp = client.get("/api/hillclimb/blueprint").json()
|
| 139 |
+
bp["competencies"][0]["prerequisites"] = ["does.not.exist"]
|
| 140 |
+
res = client.put("/api/hillclimb/blueprint", json=bp)
|
| 141 |
+
assert res.status_code == 400
|
| 142 |
+
assert "unknown prerequisite" in str(res.json())
|
tests/test_hillclimb/test_blueprint.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Blueprint & competency-graph integrity."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb.blueprint import (competency_index, downstream_map,
|
| 4 |
+
display_dimensions_for, load_blueprint,
|
| 5 |
+
seed_blueprint, validate_graph)
|
| 6 |
+
from sensorflow.hillclimb.models import Dimension, get_store
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def test_four_phases_with_full_structure():
|
| 10 |
+
bp = seed_blueprint()
|
| 11 |
+
assert [p.phase for p in bp.phases] == [1, 2, 3, 4]
|
| 12 |
+
for p in bp.phases:
|
| 13 |
+
assert p.objective
|
| 14 |
+
assert p.topics and p.skills and p.exercises and p.assessments
|
| 15 |
+
assert p.completion_criteria
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def test_marked_reconstructed_from_spec_and_seeded_editable():
|
| 19 |
+
store = get_store()
|
| 20 |
+
bp = load_blueprint(store)
|
| 21 |
+
assert bp.source == "reconstructed-from-spec"
|
| 22 |
+
# persisted as an editable data structure, not hardcoded prose
|
| 23 |
+
raw = store.get("blueprint", "active")
|
| 24 |
+
assert raw is not None and raw["source"] == "reconstructed-from-spec"
|
| 25 |
+
# editing the stored copy is what subsequent loads see
|
| 26 |
+
raw["version"] = 2
|
| 27 |
+
store.put("blueprint", "active", raw)
|
| 28 |
+
assert load_blueprint(store).version == 2
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def test_no_prerequisite_cycles_and_no_dangling_ids():
|
| 32 |
+
bp = seed_blueprint()
|
| 33 |
+
assert validate_graph(bp) == []
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def test_every_competency_tagged_with_exactly_one_dimension():
|
| 37 |
+
bp = seed_blueprint()
|
| 38 |
+
for c in bp.competencies:
|
| 39 |
+
assert isinstance(c.dimension, Dimension)
|
| 40 |
+
# all four dimensions are represented and tracked separately
|
| 41 |
+
dims = {c.dimension for c in bp.competencies}
|
| 42 |
+
assert dims == set(Dimension)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def test_key_prerequisite_edge_distributed_fundamentals_to_parallel_inference():
|
| 46 |
+
idx = competency_index(seed_blueprint())
|
| 47 |
+
assert "p2.distributed_fundamentals" in idx["p2.parallel_inference"].prerequisites
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_downstream_map_transitive():
|
| 51 |
+
bp = seed_blueprint()
|
| 52 |
+
down = downstream_map(bp)
|
| 53 |
+
# distributed fundamentals transitively unlocks the phase-4 capstone
|
| 54 |
+
assert "p4.hill_climbing" in down["p2.distributed_fundamentals"]
|
| 55 |
+
# capstone unlocks nothing
|
| 56 |
+
assert down["p4.hill_climbing"] == set()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_display_dimensions_cover_all_six():
|
| 60 |
+
bp = seed_blueprint()
|
| 61 |
+
covered = set()
|
| 62 |
+
for c in bp.competencies:
|
| 63 |
+
covered |= set(display_dimensions_for(c))
|
| 64 |
+
assert covered == {"Technical Depth", "System Design", "Execution",
|
| 65 |
+
"Leadership", "Communication", "Safety/Risk"}
|
tests/test_hillclimb/test_content.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exercise generation: required shape, seeded structural variation, canonical family."""
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from sensorflow.hillclimb.content import generate_exercise, get_exercise
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def test_exercise_has_required_shape():
|
| 9 |
+
ex = generate_exercise("p1.cusum", difficulty=2, seed=5, use_llm=False)
|
| 10 |
+
assert ex.competency_id == "p1.cusum"
|
| 11 |
+
assert ex.difficulty == 2
|
| 12 |
+
assert ex.prerequisites == ["p1.statistical_evaluation"]
|
| 13 |
+
assert len(ex.scenario) > 50
|
| 14 |
+
assert ex.expected_reasoning
|
| 15 |
+
assert ex.evaluation_rubric
|
| 16 |
+
assert ex.common_failure_modes
|
| 17 |
+
assert ex.follow_up_questions
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_structurally_different_regeneration_for_same_competency():
|
| 21 |
+
a = generate_exercise("p1.regression_detection", seed=101, use_llm=False)
|
| 22 |
+
b = generate_exercise("p1.regression_detection", seed=202, use_llm=False)
|
| 23 |
+
assert a.exercise_id != b.exercise_id
|
| 24 |
+
assert a.scenario != b.scenario # different slot values / variant
|
| 25 |
+
assert a.template_id != b.template_id # seeded template stamp differs
|
| 26 |
+
# same competency, same family — the rubric skeleton stays comparable
|
| 27 |
+
assert a.competency_id == b.competency_id
|
| 28 |
+
assert a.family == b.family == "offline_shadow"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def test_same_seed_is_deterministic():
|
| 32 |
+
a = generate_exercise("p2.parallel_inference", seed=7, use_llm=False)
|
| 33 |
+
b = generate_exercise("p2.parallel_inference", seed=7, use_llm=False)
|
| 34 |
+
assert a.scenario == b.scenario
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_canonical_offline_shadow_family_rubric_distinguishes_seven_causes():
|
| 38 |
+
ex = generate_exercise("p1.regression_detection", seed=1, use_llm=False)
|
| 39 |
+
criteria = " | ".join(i.criterion.lower() for i in ex.evaluation_rubric)
|
| 40 |
+
for cause in ["metric definition", "distribution shift", "sampling bias",
|
| 41 |
+
"feature", "serving", "noise", "true regression"]:
|
| 42 |
+
assert cause in criteria, f"rubric missing cause: {cause}"
|
| 43 |
+
# cross-linked to the live RCA workbench (read-only pointer)
|
| 44 |
+
assert ex.linked_tool is not None
|
| 45 |
+
assert ex.linked_tool["page"] == "rca"
|
| 46 |
+
assert ex.linked_tool["api"] == "/api/rca"
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def test_every_competency_can_generate(isolated_store):
|
| 50 |
+
from sensorflow.hillclimb.blueprint import load_blueprint
|
| 51 |
+
bp = load_blueprint(isolated_store)
|
| 52 |
+
for comp in bp.competencies:
|
| 53 |
+
ex = generate_exercise(comp.id, seed=3, use_llm=False)
|
| 54 |
+
assert ex.evaluation_rubric and ex.scenario
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def test_exercise_persisted_and_retrievable():
|
| 58 |
+
ex = generate_exercise("p3.hiring", seed=9, use_llm=False)
|
| 59 |
+
again = get_exercise(ex.exercise_id)
|
| 60 |
+
assert again is not None and again.scenario == ex.scenario
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_unknown_competency_rejected():
|
| 64 |
+
with pytest.raises(ValueError):
|
| 65 |
+
generate_exercise("p9.nope", use_llm=False)
|
tests/test_hillclimb/test_design_lab.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Design-lab grader: good vs broken architectures."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb.design_lab import (Component, DesignSubmission, Edge,
|
| 4 |
+
grade_submission)
|
| 5 |
+
|
| 6 |
+
RATIONALES = {
|
| 7 |
+
"Failure-mining trigger criteria": (
|
| 8 |
+
"Monitoring computes slice metrics and alerts when drift exceeds 2 sigma; scalability comes "
|
| 9 |
+
"from partitioned stream consumers handling 20000 qps with 30% headroom."),
|
| 10 |
+
"Label routing & quality control": (
|
| 11 |
+
"Redundant dual-region labeling queues (replicated, failover in under 5 minutes) with "
|
| 12 |
+
"observability dashboards, logs and traces on every stage; cost is controlled by tiered "
|
| 13 |
+
"storage and spot instances."),
|
| 14 |
+
"Loop latency (failure → fixed model)": (
|
| 15 |
+
"End-to-end loop latency target is 7 days; p99 inference latency stays under 100 ms. "
|
| 16 |
+
"The tradeoff: faster loops cost more compute, at the expense of budget."),
|
| 17 |
+
"How you detect the loop itself breaking": (
|
| 18 |
+
"A dead letter queue plus a heartbeat monitor covers failure handling; if mining stalls we "
|
| 19 |
+
"degrade gracefully to weekly batch replay and recover from checkpoints."),
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _good_submission():
|
| 24 |
+
comps = [
|
| 25 |
+
Component(id="src", type="source", name="Fleet frames", note="2 PB/week"),
|
| 26 |
+
Component(id="inf1", type="inference", name="Serving A", note="10000 qps"),
|
| 27 |
+
Component(id="inf2", type="inference", name="Serving B", note="10000 qps"),
|
| 28 |
+
Component(id="mon", type="monitoring", name="Slice monitor"),
|
| 29 |
+
Component(id="fb", type="feedback", name="Failure miner"),
|
| 30 |
+
Component(id="train", type="training", name="Retrain"),
|
| 31 |
+
Component(id="ev", type="eval", name="Eval gate"),
|
| 32 |
+
]
|
| 33 |
+
edges = [Edge(source="src", target="inf1"), Edge(source="src", target="inf2"),
|
| 34 |
+
Edge(source="inf1", target="mon"), Edge(source="inf2", target="mon"),
|
| 35 |
+
Edge(source="mon", target="fb"), Edge(source="fb", target="train"),
|
| 36 |
+
Edge(source="train", target="ev"), Edge(source="ev", target="inf1"),
|
| 37 |
+
Edge(source="ev", target="inf2")]
|
| 38 |
+
return DesignSubmission(challenge_id="feedback_loop", components=comps,
|
| 39 |
+
edges=edges, rationales=RATIONALES)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _broken_submission():
|
| 43 |
+
comps = [
|
| 44 |
+
Component(id="src", type="source", name="Fleet frames"),
|
| 45 |
+
Component(id="inf", type="inference", name="Serving"),
|
| 46 |
+
Component(id="stor", type="storage", name="Orphan bucket"), # never connected
|
| 47 |
+
]
|
| 48 |
+
edges = [Edge(source="src", target="inf")]
|
| 49 |
+
return DesignSubmission(challenge_id="feedback_loop", components=comps,
|
| 50 |
+
edges=edges, rationales={"why": "it works"})
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_good_architecture_grades_well():
|
| 54 |
+
grade = grade_submission(_good_submission(), save_evidence=False)
|
| 55 |
+
s = grade.structural
|
| 56 |
+
assert s.missing_stages == []
|
| 57 |
+
assert s.orphan_components == []
|
| 58 |
+
assert s.feedback_loop_closed
|
| 59 |
+
assert s.capacity_math_found and s.capacity_quotes
|
| 60 |
+
assert grade.overall_score >= 3
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_broken_architecture_reports_specific_gaps():
|
| 64 |
+
grade = grade_submission(_broken_submission(), save_evidence=False)
|
| 65 |
+
s = grade.structural
|
| 66 |
+
assert "monitoring" in " ".join(s.missing_stages)
|
| 67 |
+
assert "Orphan bucket" in s.orphan_components
|
| 68 |
+
assert not s.feedback_loop_closed
|
| 69 |
+
assert not s.capacity_math_found
|
| 70 |
+
joined = " ".join(grade.gaps).lower()
|
| 71 |
+
assert "missing required stage" in joined
|
| 72 |
+
assert "orphan" in joined
|
| 73 |
+
assert "feedback loop is not closed" in joined
|
| 74 |
+
assert "capacity" in joined
|
| 75 |
+
assert grade.overall_score < grade_submission(_good_submission(), save_evidence=False).overall_score
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_spof_detection():
|
| 79 |
+
# single inference node shared by everything upstream/downstream = articulation point
|
| 80 |
+
comps = [
|
| 81 |
+
Component(id="src", type="source"), Component(id="inf", type="inference"),
|
| 82 |
+
Component(id="mon", type="monitoring"), Component(id="fb", type="feedback"),
|
| 83 |
+
Component(id="train", type="training"), Component(id="ev", type="eval"),
|
| 84 |
+
]
|
| 85 |
+
edges = [Edge(source="src", target="inf"), Edge(source="inf", target="mon"),
|
| 86 |
+
Edge(source="mon", target="fb"), Edge(source="fb", target="train"),
|
| 87 |
+
Edge(source="train", target="ev"), Edge(source="ev", target="inf")]
|
| 88 |
+
grade = grade_submission(DesignSubmission(challenge_id="feedback_loop", components=comps,
|
| 89 |
+
edges=edges, rationales={}), save_evidence=False)
|
| 90 |
+
assert "inf" in grade.structural.single_points_of_failure
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def test_per_dimension_scores_with_rationale_evidence():
|
| 94 |
+
grade = grade_submission(_good_submission(), save_evidence=False)
|
| 95 |
+
dims = {d.dimension: d for d in grade.dimension_grades}
|
| 96 |
+
assert set(dims) == {"scalability", "reliability", "latency", "cost",
|
| 97 |
+
"observability", "failure_handling", "tradeoffs"}
|
| 98 |
+
assert dims["scalability"].score >= 3 and dims["scalability"].evidence
|
| 99 |
+
assert dims["tradeoffs"].score >= 2
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def test_grading_stores_evidence_artifact(isolated_store):
|
| 103 |
+
grade = grade_submission(_good_submission(), store=isolated_store, save_evidence=True)
|
| 104 |
+
assert grade.evidence_id
|
| 105 |
+
raw = isolated_store.get("evidence", grade.evidence_id)
|
| 106 |
+
assert raw["artifact_type"] == "design_submission"
|
| 107 |
+
assert "p2.feedback_loops" in raw["competency_ids"]
|
tests/test_hillclimb/test_evaluate.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluation contract, evidence-required rule, anti-gaming, coaching specificity."""
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
from pydantic import ValidationError
|
| 5 |
+
|
| 6 |
+
from sensorflow.hillclimb import coach
|
| 7 |
+
from sensorflow.hillclimb.content import generate_exercise
|
| 8 |
+
from sensorflow.hillclimb.evaluate import rule_based_evaluate
|
| 9 |
+
from sensorflow.hillclimb.models import EvaluationResult
|
| 10 |
+
|
| 11 |
+
from .conftest import (CONCISE_PRECISE_ANSWER, STRONG_TECH_ANSWER, VERBOSE_WAFFLE,
|
| 12 |
+
WEAK_ANSWER)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _exercise(seed=1):
|
| 16 |
+
return generate_exercise("p1.regression_detection", seed=seed, use_llm=False)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_contract_schema_enforced():
|
| 20 |
+
with pytest.raises(ValidationError):
|
| 21 |
+
EvaluationResult(competency="x", score=0, confidence=0.5)
|
| 22 |
+
with pytest.raises(ValidationError):
|
| 23 |
+
EvaluationResult(competency="x", score=6, confidence=0.5)
|
| 24 |
+
with pytest.raises(ValidationError):
|
| 25 |
+
EvaluationResult(competency="x", score=3, confidence=1.5)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def test_strong_answer_scores_high_with_quoted_evidence():
|
| 29 |
+
result = rule_based_evaluate(_exercise(), STRONG_TECH_ANSWER)
|
| 30 |
+
assert result.score >= 4
|
| 31 |
+
assert result.evidence, "no score without evidence"
|
| 32 |
+
# evidence quotes the user's own statements
|
| 33 |
+
for quote in result.evidence:
|
| 34 |
+
assert quote.rstrip("…")[:40].lower() in STRONG_TECH_ANSWER.lower()
|
| 35 |
+
assert result.follow_up_question in _exercise().follow_up_questions
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_no_score_without_evidence():
|
| 39 |
+
result = rule_based_evaluate(_exercise(), WEAK_ANSWER)
|
| 40 |
+
assert result.score == 1
|
| 41 |
+
assert result.evidence == []
|
| 42 |
+
empty = rule_based_evaluate(_exercise(), "")
|
| 43 |
+
assert empty.score == 1
|
| 44 |
+
# the rule as an invariant: any score above 1 must carry evidence
|
| 45 |
+
for answer in [STRONG_TECH_ANSWER, CONCISE_PRECISE_ANSWER, VERBOSE_WAFFLE, WEAK_ANSWER]:
|
| 46 |
+
r = rule_based_evaluate(_exercise(), answer)
|
| 47 |
+
if r.score > 1:
|
| 48 |
+
assert r.evidence
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def test_anti_gaming_verbose_waffle_scores_below_concise_precise():
|
| 52 |
+
ex = _exercise()
|
| 53 |
+
waffle = rule_based_evaluate(ex, VERBOSE_WAFFLE)
|
| 54 |
+
concise = rule_based_evaluate(ex, CONCISE_PRECISE_ANSWER)
|
| 55 |
+
assert len(VERBOSE_WAFFLE) > 4 * len(CONCISE_PRECISE_ANSWER)
|
| 56 |
+
assert waffle.score < concise.score, (
|
| 57 |
+
f"verbosity must not raise scores: waffle={waffle.score} concise={concise.score}")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def test_misconception_detection():
|
| 61 |
+
ex = _exercise()
|
| 62 |
+
r = rule_based_evaluate(ex, "This is definitely a true regression, no need to test more. "
|
| 63 |
+
"Offline eval is ground truth anyway.")
|
| 64 |
+
assert r.misconceptions
|
| 65 |
+
assert any("significance" in m.lower() or "premature" in m.lower() for m in r.misconceptions)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_missing_evidence_lists_specific_gaps():
|
| 69 |
+
ex = _exercise()
|
| 70 |
+
r = rule_based_evaluate(ex, CONCISE_PRECISE_ANSWER)
|
| 71 |
+
joined = " ".join(r.missing_evidence).lower()
|
| 72 |
+
assert "quantified" in joined or "tradeoff" in joined
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def test_coaching_is_specific_not_generic():
|
| 76 |
+
ex = _exercise()
|
| 77 |
+
r = rule_based_evaluate(ex, CONCISE_PRECISE_ANSWER)
|
| 78 |
+
feedback = coach.build_feedback(ex, r)
|
| 79 |
+
# references what the answer DID establish...
|
| 80 |
+
assert "establishes" in feedback
|
| 81 |
+
# ...and points at concrete rubric gaps, not platitudes
|
| 82 |
+
assert "Missing:" in feedback or "quantified" in feedback.lower()
|
| 83 |
+
# recommended action mentions the competency / retry semantics
|
| 84 |
+
assert r.recommended_action
|
| 85 |
+
assert "p1.regression_detection" in r.recommended_action or "exercise" in r.recommended_action
|
tests/test_hillclimb/test_interviewer.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Interviewer adaptivity: sequencing must depend on evaluations, no fixed list."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb.content import get_exercise
|
| 4 |
+
from sensorflow.hillclimb.interviewer import (end_interview, start_interview,
|
| 5 |
+
submit_answer)
|
| 6 |
+
|
| 7 |
+
from .conftest import STRONG_LEADERSHIP_ANSWER, WEAK_ANSWER
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def test_weak_answer_triggers_targeted_probe(isolated_store):
|
| 11 |
+
s = start_interview("management", seed=5, store=isolated_store)
|
| 12 |
+
first = s.turns[0]
|
| 13 |
+
s = submit_answer(s.session_id, WEAK_ANSWER, use_llm=False, store=isolated_store)
|
| 14 |
+
nxt = s.turns[-1]
|
| 15 |
+
assert s.turns[0].evaluation.score <= 2
|
| 16 |
+
assert nxt.question_type == "probe"
|
| 17 |
+
# the probe comes from the exercise's own follow_up_questions
|
| 18 |
+
ex = get_exercise(first.exercise_id, isolated_store)
|
| 19 |
+
assert nxt.question in ex.follow_up_questions
|
| 20 |
+
assert nxt.competency_id == first.competency_id
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_strong_answer_escalates(isolated_store):
|
| 24 |
+
s = start_interview("management", seed=5, store=isolated_store)
|
| 25 |
+
s = submit_answer(s.session_id, STRONG_LEADERSHIP_ANSWER, use_llm=False, store=isolated_store)
|
| 26 |
+
assert s.turns[0].evaluation.score >= 4
|
| 27 |
+
nxt = s.turns[-1]
|
| 28 |
+
assert nxt.question_type == "escalate"
|
| 29 |
+
assert nxt.difficulty == 3 # difficulty went up
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_sequencing_depends_on_answer_quality(isolated_store):
|
| 33 |
+
weak = start_interview("management", seed=5, store=isolated_store)
|
| 34 |
+
strong = start_interview("management", seed=5, store=isolated_store)
|
| 35 |
+
# identical opening question (same seed) ...
|
| 36 |
+
assert weak.turns[0].question == strong.turns[0].question
|
| 37 |
+
weak = submit_answer(weak.session_id, WEAK_ANSWER, use_llm=False, store=isolated_store)
|
| 38 |
+
strong = submit_answer(strong.session_id, STRONG_LEADERSHIP_ANSWER, use_llm=False,
|
| 39 |
+
store=isolated_store)
|
| 40 |
+
# ... but different evaluations produce different next questions
|
| 41 |
+
assert weak.turns[-1].question != strong.turns[-1].question
|
| 42 |
+
assert weak.turns[-1].question_type != strong.turns[-1].question_type
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def test_probe_limit_then_moves_on(isolated_store):
|
| 46 |
+
s = start_interview("management", seed=5, store=isolated_store)
|
| 47 |
+
s = submit_answer(s.session_id, WEAK_ANSWER, use_llm=False, store=isolated_store)
|
| 48 |
+
s = submit_answer(s.session_id, WEAK_ANSWER, use_llm=False, store=isolated_store)
|
| 49 |
+
assert s.turns[-1].question_type == "probe"
|
| 50 |
+
s = submit_answer(s.session_id, WEAK_ANSWER, use_llm=False, store=isolated_store)
|
| 51 |
+
assert s.turns[-1].question_type == "advance" # max 2 probes, then new competency
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_transcript_saved_as_evidence(isolated_store):
|
| 55 |
+
s = start_interview("technical", seed=8, store=isolated_store)
|
| 56 |
+
s = submit_answer(s.session_id, "Because dynamic batching amortizes overhead it can fail "
|
| 57 |
+
"under overload; I would keep 30% headroom as the tradeoff.",
|
| 58 |
+
use_llm=False, store=isolated_store)
|
| 59 |
+
s = end_interview(s.session_id, store=isolated_store)
|
| 60 |
+
assert s.status == "complete" and s.evidence_id
|
| 61 |
+
raw = isolated_store.get("evidence", s.evidence_id)
|
| 62 |
+
assert raw["artifact_type"] == "interview_transcript"
|
| 63 |
+
assert raw["payload"]["transcript"]
|
| 64 |
+
assert raw["competency_ids"]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def test_modes_restrict_competency_pool(isolated_store):
|
| 68 |
+
tech = start_interview("technical", seed=2, store=isolated_store)
|
| 69 |
+
mgmt = start_interview("management", seed=2, store=isolated_store)
|
| 70 |
+
assert tech.turns[0].competency_id.split(".")[0] in ("p1", "p2")
|
| 71 |
+
assert mgmt.turns[0].competency_id.startswith("p3")
|
tests/test_hillclimb/test_journey.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Journey state machine incl. FAIL -> REMEDIATION routing to the diagnosed prerequisite."""
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from sensorflow.hillclimb import adaptive, readiness
|
| 6 |
+
from sensorflow.hillclimb.models import Attempt, EvaluationResult, JourneyState
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _seed(store, cid, score):
|
| 10 |
+
for kind in ("diagnostic", "exercise"):
|
| 11 |
+
readiness.record_attempt(
|
| 12 |
+
Attempt(user_id="default", competency_id=cid, kind=kind, exercise_id="x",
|
| 13 |
+
responses={},
|
| 14 |
+
evaluation=EvaluationResult(competency=cid, score=score, confidence=0.8,
|
| 15 |
+
evidence=["q"])), store)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def test_happy_path_transitions(isolated_store):
|
| 19 |
+
j = adaptive.get_journey(store=isolated_store)
|
| 20 |
+
assert j.state == JourneyState.NOT_STARTED
|
| 21 |
+
j = adaptive.advance_journey("default", "start_diagnostic", store=isolated_store)
|
| 22 |
+
assert j.state == JourneyState.DIAGNOSTIC
|
| 23 |
+
j = adaptive.advance_journey("default", "diagnostic_complete", store=isolated_store)
|
| 24 |
+
assert j.state == JourneyState.LEARNING
|
| 25 |
+
j = adaptive.advance_journey("default", "begin_practice",
|
| 26 |
+
competency_id="p1.precision_recall", store=isolated_store)
|
| 27 |
+
assert j.state == JourneyState.PRACTICE
|
| 28 |
+
j = adaptive.advance_journey("default", "request_assessment", store=isolated_store)
|
| 29 |
+
assert j.state == JourneyState.ASSESSMENT
|
| 30 |
+
j = adaptive.advance_journey("default", "assessment_result", passed=True,
|
| 31 |
+
competency_id="p1.precision_recall", store=isolated_store)
|
| 32 |
+
assert j.state == JourneyState.LEARNING # PASS -> next
|
| 33 |
+
assert j.current_competency != "p1.precision_recall"
|
| 34 |
+
assert j.history and j.history[-1]["passed"] is True
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_invalid_transitions_rejected(isolated_store):
|
| 38 |
+
with pytest.raises(ValueError):
|
| 39 |
+
adaptive.advance_journey("default", "request_assessment", store=isolated_store)
|
| 40 |
+
with pytest.raises(ValueError):
|
| 41 |
+
adaptive.advance_journey("default", "assessment_result", passed=True, store=isolated_store)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def test_fail_routes_remediation_to_diagnosed_prerequisite(isolated_store):
|
| 45 |
+
# weak prerequisite: distributed fundamentals; the advanced item fails
|
| 46 |
+
_seed(isolated_store, "p2.distributed_fundamentals", 2)
|
| 47 |
+
adaptive.advance_journey("default", "start_diagnostic", store=isolated_store)
|
| 48 |
+
adaptive.advance_journey("default", "diagnostic_complete", store=isolated_store)
|
| 49 |
+
adaptive.advance_journey("default", "begin_practice",
|
| 50 |
+
competency_id="p2.parallel_inference", store=isolated_store)
|
| 51 |
+
adaptive.advance_journey("default", "request_assessment", store=isolated_store)
|
| 52 |
+
j = adaptive.advance_journey("default", "assessment_result", passed=False,
|
| 53 |
+
competency_id="p2.parallel_inference", store=isolated_store)
|
| 54 |
+
assert j.state == JourneyState.REMEDIATION
|
| 55 |
+
# backward routing: the diagnosed MISSING PREREQUISITE, not the failed item
|
| 56 |
+
assert j.remediation_target == "p2.distributed_fundamentals"
|
| 57 |
+
|
| 58 |
+
j = adaptive.advance_journey("default", "remediation_complete", store=isolated_store)
|
| 59 |
+
assert j.state == JourneyState.REASSESS
|
| 60 |
+
j = adaptive.advance_journey("default", "assessment_result", passed=True,
|
| 61 |
+
competency_id="p2.parallel_inference", store=isolated_store)
|
| 62 |
+
assert j.state == JourneyState.LEARNING
|
| 63 |
+
assert j.remediation_target is None
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def test_fail_without_prerequisites_remediates_in_place(isolated_store):
|
| 67 |
+
adaptive.advance_journey("default", "start_diagnostic", store=isolated_store)
|
| 68 |
+
adaptive.advance_journey("default", "diagnostic_complete", store=isolated_store)
|
| 69 |
+
adaptive.advance_journey("default", "begin_practice",
|
| 70 |
+
competency_id="p1.precision_recall", store=isolated_store)
|
| 71 |
+
adaptive.advance_journey("default", "request_assessment", store=isolated_store)
|
| 72 |
+
j = adaptive.advance_journey("default", "assessment_result", passed=False,
|
| 73 |
+
competency_id="p1.precision_recall", store=isolated_store)
|
| 74 |
+
assert j.state == JourneyState.REMEDIATION
|
| 75 |
+
assert j.remediation_target == "p1.precision_recall"
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_reassess_fail_loops_back_to_remediation(isolated_store):
|
| 79 |
+
_seed(isolated_store, "p2.distributed_fundamentals", 2)
|
| 80 |
+
adaptive.advance_journey("default", "start_diagnostic", store=isolated_store)
|
| 81 |
+
adaptive.advance_journey("default", "diagnostic_complete", store=isolated_store)
|
| 82 |
+
adaptive.advance_journey("default", "begin_practice",
|
| 83 |
+
competency_id="p2.parallel_inference", store=isolated_store)
|
| 84 |
+
adaptive.advance_journey("default", "request_assessment", store=isolated_store)
|
| 85 |
+
adaptive.advance_journey("default", "assessment_result", passed=False,
|
| 86 |
+
competency_id="p2.parallel_inference", store=isolated_store)
|
| 87 |
+
adaptive.advance_journey("default", "remediation_complete", store=isolated_store)
|
| 88 |
+
j = adaptive.advance_journey("default", "assessment_result", passed=False,
|
| 89 |
+
competency_id="p2.parallel_inference", store=isolated_store)
|
| 90 |
+
assert j.state == JourneyState.REMEDIATION
|
tests/test_hillclimb/test_readiness.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Readiness matrix, state derivation, and leverage-based bottleneck analysis."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb import readiness
|
| 4 |
+
from sensorflow.hillclimb.blueprint import load_blueprint
|
| 5 |
+
from sensorflow.hillclimb.models import Attempt, EvaluationResult, ReadinessState
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _attempt(cid, score, kind, user="default"):
|
| 9 |
+
return Attempt(user_id=user, competency_id=cid, kind=kind,
|
| 10 |
+
exercise_id="x", responses={},
|
| 11 |
+
evaluation=EvaluationResult(competency=cid, score=score, confidence=0.8,
|
| 12 |
+
evidence=["quoted statement"]))
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _seed_scores(store, cid, score, user="default"):
|
| 16 |
+
readiness.record_attempt(_attempt(cid, score, "diagnostic", user), store)
|
| 17 |
+
readiness.record_attempt(_attempt(cid, score, "exercise", user), store)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_states_derive_from_scores(isolated_store):
|
| 21 |
+
_seed_scores(isolated_store, "p1.precision_recall", 5)
|
| 22 |
+
_seed_scores(isolated_store, "p1.iou", 3)
|
| 23 |
+
_seed_scores(isolated_store, "p1.cusum", 2)
|
| 24 |
+
matrix = readiness.compute_matrix(store=isolated_store)
|
| 25 |
+
assert matrix["p1.precision_recall"].readiness_state == ReadinessState.STRONG
|
| 26 |
+
assert matrix["p1.iou"].readiness_state == ReadinessState.COMPETENT
|
| 27 |
+
assert matrix["p1.cusum"].readiness_state == ReadinessState.PRACTICING
|
| 28 |
+
assert matrix["p1.ap_map"].readiness_state == ReadinessState.NOT_STARTED
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def test_needs_review_on_knowledge_application_mismatch(isolated_store):
|
| 32 |
+
readiness.record_attempt(_attempt("p1.iou", 5, "diagnostic"), isolated_store)
|
| 33 |
+
readiness.record_attempt(_attempt("p1.iou", 2, "exercise"), isolated_store)
|
| 34 |
+
matrix = readiness.compute_matrix(store=isolated_store)
|
| 35 |
+
assert matrix["p1.iou"].readiness_state == ReadinessState.NEEDS_REVIEW
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_scores_tracked_separately_never_collapsed(isolated_store):
|
| 39 |
+
readiness.record_attempt(_attempt("p1.iou", 5, "diagnostic"), isolated_store)
|
| 40 |
+
readiness.record_attempt(_attempt("p1.iou", 2, "exercise"), isolated_store)
|
| 41 |
+
r = readiness.compute_matrix(store=isolated_store)["p1.iou"]
|
| 42 |
+
assert r.knowledge_score == 5.0
|
| 43 |
+
assert r.application_score == 2.0
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def test_bottleneck_is_blocking_prerequisite_not_min_score(isolated_store):
|
| 47 |
+
bp = load_blueprint(isolated_store)
|
| 48 |
+
weak_hub = "p2.distributed_fundamentals" # blocks much of phase 2
|
| 49 |
+
lowest = "p3.mentorship" # lowest raw score, blocks nothing
|
| 50 |
+
untouched = {"p2.petabyte_ingestion", "p2.streaming_batch", "p2.parallel_inference"}
|
| 51 |
+
for comp in bp.competencies:
|
| 52 |
+
if comp.id == weak_hub:
|
| 53 |
+
_seed_scores(isolated_store, comp.id, 2)
|
| 54 |
+
elif comp.id == lowest:
|
| 55 |
+
_seed_scores(isolated_store, comp.id, 1)
|
| 56 |
+
elif comp.id not in untouched:
|
| 57 |
+
_seed_scores(isolated_store, comp.id, 5)
|
| 58 |
+
|
| 59 |
+
bn = readiness.bottleneck_analysis(store=isolated_store)
|
| 60 |
+
assert bn is not None
|
| 61 |
+
assert bn["competency_id"] == weak_hub, (
|
| 62 |
+
f"bottleneck must be the blocking prerequisite, got {bn['competency_id']}")
|
| 63 |
+
assert bn["blocked_count"] >= 3
|
| 64 |
+
assert set(bn["blocked_competencies"]) >= untouched
|
| 65 |
+
# plain-language explanation names the leverage
|
| 66 |
+
assert "limiting" in bn["explanation"]
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def test_bottleneck_none_when_everything_competent(isolated_store):
|
| 70 |
+
bp = load_blueprint(isolated_store)
|
| 71 |
+
for comp in bp.competencies:
|
| 72 |
+
_seed_scores(isolated_store, comp.id, 5)
|
| 73 |
+
assert readiness.bottleneck_analysis(store=isolated_store) is None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_dimension_summary_covers_display_dimensions(isolated_store):
|
| 77 |
+
_seed_scores(isolated_store, "p1.precision_recall", 4)
|
| 78 |
+
dims = readiness.dimension_summary(store=isolated_store)
|
| 79 |
+
names = {d["dimension"] for d in dims}
|
| 80 |
+
assert names == {"Technical Depth", "System Design", "Execution", "Leadership",
|
| 81 |
+
"Communication", "Safety/Risk"}
|
| 82 |
+
tech = next(d for d in dims if d["dimension"] == "Technical Depth")
|
| 83 |
+
assert tech["avg_score"] > 0
|
tests/test_hillclimb/test_simulation.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Simulation: determinism, safety-floor incidents, keep/reject, debrief mapping."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb.simulation import (create_simulation, get_simulation,
|
| 4 |
+
step_simulation)
|
| 5 |
+
|
| 6 |
+
STRONG_HYP = "Investing in monitoring will improve safety by 5 points within two turns"
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _run(store, seed, moves):
|
| 10 |
+
sim = create_simulation(seed=seed, max_turns=len(moves), store=store)
|
| 11 |
+
for hyp, iid in moves:
|
| 12 |
+
sim = step_simulation(sim.sim_id, hyp, iid, store=store)
|
| 13 |
+
return sim
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def test_deterministic_given_seed(isolated_store):
|
| 17 |
+
moves = [(STRONG_HYP, "add_monitoring"),
|
| 18 |
+
("Pausing launches will raise safety", "pause_launches"),
|
| 19 |
+
("1:1s will recover team morale by 5", "team_1on1s")]
|
| 20 |
+
a = _run(isolated_store, 42, moves)
|
| 21 |
+
b = _run(isolated_store, 42, moves)
|
| 22 |
+
assert a.objective_history == b.objective_history
|
| 23 |
+
assert a.metrics == b.metrics
|
| 24 |
+
c = _run(isolated_store, 43, moves)
|
| 25 |
+
assert c.objective_history != a.objective_history
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def test_safety_floor_triggers_incident_event(isolated_store):
|
| 29 |
+
sim = create_simulation(seed=1, max_turns=6, store=isolated_store)
|
| 30 |
+
for _ in range(5):
|
| 31 |
+
sim = step_simulation(sim.sim_id, "Shipping fast will improve schedule",
|
| 32 |
+
"ship_feature_fast", store=isolated_store)
|
| 33 |
+
if any(e["type"] == "SAFETY_INCIDENT" for e in sim.events):
|
| 34 |
+
break
|
| 35 |
+
incidents = [e for e in sim.events if e["type"] == "SAFETY_INCIDENT"]
|
| 36 |
+
assert incidents, "repeated corner-cutting must breach the safety floor"
|
| 37 |
+
assert "regression escaped" in incidents[0]["detail"]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_keep_reject_reverts_previous_effects(isolated_store):
|
| 41 |
+
sim = create_simulation(seed=9, max_turns=4, store=isolated_store)
|
| 42 |
+
sim = step_simulation(sim.sim_id, "Shipping fast raises schedule", "ship_feature_fast",
|
| 43 |
+
store=isolated_store)
|
| 44 |
+
applied = sim.history[0].applied_effects
|
| 45 |
+
sim = step_simulation(sim.sim_id, "Reverting; monitoring will improve safety",
|
| 46 |
+
"add_monitoring", revert_previous=True, store=isolated_store)
|
| 47 |
+
assert sim.history[1].reverted_previous
|
| 48 |
+
assert any("Rejected previous intervention" in e for e in sim.history[1].events)
|
| 49 |
+
assert applied # effects existed and were rolled back
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def test_delayed_second_order_effects_land_later(isolated_store):
|
| 53 |
+
sim = create_simulation(seed=3, max_turns=4, store=isolated_store)
|
| 54 |
+
sim = step_simulation(sim.sim_id, "Test infra will raise reliability", "invest_test_infra",
|
| 55 |
+
store=isolated_store)
|
| 56 |
+
assert sim.pending, "delayed effects must be queued"
|
| 57 |
+
sim = step_simulation(sim.sim_id, "Waiting for infra to land", "team_1on1s", store=isolated_store)
|
| 58 |
+
sim = step_simulation(sim.sim_id, "Infra lands this turn", "team_1on1s", store=isolated_store)
|
| 59 |
+
landed = [n for t in sim.history for n in t.delayed_landed]
|
| 60 |
+
assert any("Test infra lands" in n for n in landed)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_debrief_maps_decisions_to_competency_evidence(isolated_store):
|
| 64 |
+
moves = [(STRONG_HYP, "add_monitoring"),
|
| 65 |
+
("Pausing launches will raise safety above 55", "pause_launches"),
|
| 66 |
+
("Test infra will improve reliability by 7", "invest_test_infra"),
|
| 67 |
+
("Expectation reset will improve schedule by 6", "exec_expectation_reset")]
|
| 68 |
+
sim = _run(isolated_store, 42, moves)
|
| 69 |
+
assert sim.status == "complete"
|
| 70 |
+
assert sim.debrief is not None
|
| 71 |
+
mapped = {m["competency_id"]: m for m in sim.debrief["competency_mappings"]}
|
| 72 |
+
assert "p4.hill_climbing" in mapped
|
| 73 |
+
assert mapped["p4.hill_climbing"]["verdict"] == "evidenced" # falsifiable hypotheses
|
| 74 |
+
assert "p2.reliability_tradeoffs" in mapped
|
| 75 |
+
# evidenced mappings persist Evidence artifacts
|
| 76 |
+
ev = isolated_store.where("evidence", artifact_type="simulation_debrief")
|
| 77 |
+
assert ev, "debrief must store evidence"
|
| 78 |
+
# state is retrievable
|
| 79 |
+
assert get_simulation(sim.sim_id, isolated_store).status == "complete"
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def test_vague_hypotheses_marked_as_gap(isolated_store):
|
| 83 |
+
moves = [("do stuff", "add_monitoring"), ("things", "team_1on1s"),
|
| 84 |
+
("whatever", "cut_scope"), ("meh", "exec_expectation_reset")]
|
| 85 |
+
sim = _run(isolated_store, 5, moves)
|
| 86 |
+
mapped = {m["competency_id"]: m for m in sim.debrief["competency_mappings"]}
|
| 87 |
+
assert mapped["p4.hill_climbing"]["verdict"] == "gap"
|
tests/test_hillclimb/test_star.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""STAR Story Box: diagnosis, claim-vs-evidence, competency mapping, evidence storage."""
|
| 2 |
+
|
| 3 |
+
from sensorflow.hillclimb.star import diagnose_story
|
| 4 |
+
|
| 5 |
+
GOOD_STORY = (
|
| 6 |
+
"When I joined, the team of 12 engineers owned a perception system whose release process was "
|
| 7 |
+
"failing under a hard quarterly deadline. I was asked to fix the release pipeline. "
|
| 8 |
+
"I decided to build a regression gate, and instead of pausing all launches I chose a staged "
|
| 9 |
+
"rollout; the tech lead disagreed, and I convinced him with data from the last three incidents. "
|
| 10 |
+
"As a result, deploy failures dropped from 14% to 3% within two quarters, and since then we "
|
| 11 |
+
"have sustained the improvement with weekly monitoring."
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
VAGUE_STORY = (
|
| 15 |
+
"My team had some problems with quality. We worked together on improving things and "
|
| 16 |
+
"significantly improved performance. Everyone was happier and things got much better overall."
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_good_story_diagnosed_into_components():
|
| 21 |
+
d = diagnose_story(GOOD_STORY, save_evidence=False)
|
| 22 |
+
by_label = {c.component: c for c in d.components}
|
| 23 |
+
assert by_label["S"].present and by_label["A"].present and by_label["R"].present
|
| 24 |
+
assert d.overall_score >= 4
|
| 25 |
+
passed = {c.check for c in d.checks if c.passed}
|
| 26 |
+
assert {"personal_ownership", "measurable_outcome", "influence_disagreement",
|
| 27 |
+
"constraints", "follow_through"} <= passed
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def test_claim_vs_evidence_detector():
|
| 31 |
+
vague = diagnose_story(VAGUE_STORY, save_evidence=False)
|
| 32 |
+
unquantified = [f for f in vague.claim_flags if f.kind == "unquantified_claim"]
|
| 33 |
+
assert unquantified, "must flag 'significantly improved performance' style claims"
|
| 34 |
+
assert any("metric" in f.detail or "before" in f.detail for f in unquantified)
|
| 35 |
+
|
| 36 |
+
good = diagnose_story(GOOD_STORY, save_evidence=False)
|
| 37 |
+
measurable = [f for f in good.claim_flags if f.kind == "measurable_evidence"]
|
| 38 |
+
assert measurable, "quantified before/after claims must be recognized as evidence"
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def test_unquantified_claims_cap_score_and_produce_strengthen_prompts():
|
| 42 |
+
d = diagnose_story(VAGUE_STORY, save_evidence=False)
|
| 43 |
+
assert d.overall_score <= 3
|
| 44 |
+
assert any("Strengthen evidence" in c for c in d.coaching)
|
| 45 |
+
# weak ownership called out specifically
|
| 46 |
+
ownership = next(c for c in d.checks if c.check == "personal_ownership")
|
| 47 |
+
assert not ownership.passed
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_story_maps_to_phase3_competencies():
|
| 51 |
+
d = diagnose_story(GOOD_STORY, save_evidence=False)
|
| 52 |
+
ids = {c["competency_id"] for c in d.competencies}
|
| 53 |
+
assert "p3.conflict_resolution" in ids # "disagreed"
|
| 54 |
+
assert all(cid.startswith("p3.") for cid in ids)
|
| 55 |
+
assert all(c["reason"] for c in d.competencies)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def test_saved_as_evidence_artifact(isolated_store):
|
| 59 |
+
d = diagnose_story(GOOD_STORY, user_id="u1", save_evidence=True, store=isolated_store)
|
| 60 |
+
assert d.evidence_id
|
| 61 |
+
raw = isolated_store.get("evidence", d.evidence_id)
|
| 62 |
+
assert raw["artifact_type"] == "star_story"
|
| 63 |
+
assert raw["competency_ids"]
|
| 64 |
+
assert raw["score"] == d.overall_score
|