/** * Statistics across a sample of episodes, for the Action Insights and Filtering tabs: which * episodes to read, and what is computed from them. Nothing here fetches; the loader in * fetch-data.ts reads the episodes and hands them over as flat typed arrays. */ import { CHART_CONFIG } from "@/utils/constants"; export type LowMovementEpisode = { episodeIndex: number; totalMovement: number; }; export type AggVelocityStat = { name: string; std: number; // normalized by motor range maxAbs: number; // normalized by motor range /** Counts of Δa over `[lo, hi]`, which is symmetric about zero; outliers are in the end bins. */ bins: number[]; lo: number; // normalized by motor range hi: number; // normalized by motor range motorRange: number; inactive?: boolean; // true if p95(|Δa|) < 0.1% of motor range in every episode discrete?: boolean; // true if motor has very few unique values (e.g. open/close gripper) }; export type AggAutocorrelation = { chartData: Record[]; suggestedChunk: number | null; shortKeys: string[]; }; export type SpeedDistEntry = { episodeIndex: number; speed: number; }; export type AggAlignment = { ccData: { lag: number; max: number; mean: number; min: number }[]; meanPeakLag: number; meanPeakCorr: number; maxPeakLag: number; maxPeakCorr: number; minPeakLag: number; minPeakCorr: number; lagRangeMin: number; lagRangeMax: number; numPairs: number; }; export type JerkyEpisode = { episodeIndex: number; meanAbsDelta: number; }; export type CrossEpisodeVarianceData = { actionNames: string[]; timeBins: number[]; variance: number[][]; numEpisodes: number; lowMovementEpisodes: LowMovementEpisode[]; aggVelocity: AggVelocityStat[]; aggAutocorrelation: AggAutocorrelation | null; speedDistribution: SpeedDistEntry[]; jerkyEpisodes: JerkyEpisode[]; aggAlignment: AggAlignment | null; }; /** One sampled episode, every frame of it. */ export type EpisodeSeries = { index: number; /** Frames in the episode. */ length: number; /** Row-major: frame `t`, dimension `d` at `[t * actionDim + d]`. */ actions: Float32Array; /** Same layout with `stateDim`; null when the episode has no usable state. */ states: Float32Array | null; }; const DISCRETE_THRESHOLD = 4; // ≤ this many unique values → discrete motor const ACTIVITY_THRESHOLD = 0.001; // 0.1% of motor range /** Odd, so the middle bin is "no change". */ const VELOCITY_BINS = 31; /** The share of changes the velocity histogram's range holds; the rest sit in its end bins. */ const VELOCITY_REACH_QUANTILE = 0.995; const FINE_BINS = 1024; const MAX_ACF_LAG = 100; const ALIGNMENT_MAX_LAG = 30; /** * Frames of one episode that the lag statistics (autocorrelation, alignment) run over. They cost * frames × lags × dimensions, so a longer episode contributes this many consecutive frames from * its middle: plenty for lags of up to 100 steps. Consecutive, not every n-th: a lag or a * frame-to-frame change measured across skipped frames is no longer in frames, which is what * subsampling long episodes used to do to these numbers. */ export const LAG_WINDOW_FRAMES = 1000; function shortName(key: string): string { const parts = key.split(CHART_CONFIG.SERIES_NAME_DELIMITER); return parts.length > 1 ? parts[parts.length - 1] : key; } function evenIndices(length: number, count: number): number[] { if (count >= length) return Array.from({ length }, (_, i) => i); if (count <= 1) return length > 0 ? [0] : []; const picked = new Set(); for (let i = 0; i < count; i++) { picked.add(Math.round((i * (length - 1)) / (count - 1))); } return [...picked].sort((a, b) => a - b); } export type SampleCandidate = { /** The data file holding the episode. */ file: string; /** Row offsets inside that file; `toRow` is exclusive. */ fromRow: number; toRow: number; }; /** * Picks the episodes the analysis reads: at most `maxEpisodes`, spread over the dataset, from files * that together hold at most `rowBudget` rows. * * A v3.0 data file is read a whole row group at a time, so an episode costs its file, not its own * rows. Spreading 120 episodes evenly over a dataset of 100 files used to download all 100. Here the * files are chosen first (evenly spaced, as many as the budget allows) and the episodes are drawn * from those, so the download is bounded whatever the dataset's size. */ export function planEpisodeSample( episodes: T[], maxEpisodes: number, rowBudget: number, ): T[] { const files = new Map(); for (const episode of episodes) { let file = files.get(episode.file); if (!file) files.set(episode.file, (file = { episodes: [], rows: 0 })); file.episodes.push(episode); file.rows = Math.max(file.rows, episode.toRow); } const all = [...files.values()]; const rowsOf = (indices: number[]) => indices.reduce((sum, i) => sum + all[i].rows, 0); // As many evenly spaced files as fit the budget; never fewer than one. let chosen = evenIndices(all.length, Math.min(all.length, maxEpisodes)); while (chosen.length > 1 && rowsOf(chosen) > rowBudget) { chosen = evenIndices(all.length, chosen.length - 1); } // One file holding a single episode is not a sample: the analysis compares episodes. if ( chosen.length === 1 && all.length > 1 && all[chosen[0]].episodes.length < 2 ) { chosen = evenIndices(all.length, 2); } const sample: T[] = []; chosen.forEach((fileIndex, position) => { const file = all[fileIndex]; // The episode quota is shared out over the files still to come, so one short file's unused // share goes to the later ones. const quota = Math.ceil( (maxEpisodes - sample.length) / (chosen.length - position), ); for (const i of evenIndices(file.episodes.length, quota)) { sample.push(file.episodes[i]); } }); return sample; } /** The consecutive frames `[start, end)` of an episode that the lag statistics use. */ function lagWindow(length: number): [number, number] { if (length <= LAG_WINDOW_FRAMES) return [0, length]; const start = Math.floor((length - LAG_WINDOW_FRAMES) / 2); return [start, start + LAG_WINDOW_FRAMES]; } function median(sorted: number[]): number { return sorted[Math.floor(sorted.length / 2)]; } export type CrossEpisodeInput = { episodes: EpisodeSeries[]; actionDim: number; stateDim: number; /** One per action dimension, `feature | name`. */ actionNames: string[]; /** One per state dimension, the bare name. */ stateNames: string[]; fps: number; numTimeBins?: number; }; /** What the lag statistics need from the first pass. */ type Activity = { /** `[e * actionDim + d]`: 1 where dimension d moves in episode e. */ active: Uint8Array; /** The consecutive frames of each episode that the lag statistics use. */ windows: [number, number][]; }; type BaseStats = Omit< CrossEpisodeVarianceData, "aggAutocorrelation" | "aggAlignment" >; /** Per motor, over the whole sample: its range, and where it is moving. */ type MotorActivity = { /** max − min of each dimension, or 1 for a dimension that never changes. */ motorRanges: Float64Array; /** Dimensions taking only a few distinct values (a gripper that is open or closed). */ discrete: boolean[]; /** `[e * D + d]`: 1 where dimension d moves in episode e. */ active: Uint8Array; /** Dimensions that move in at least one episode. */ globallyActive: Uint8Array; }; function motorActivity(episodes: EpisodeSeries[], D: number): MotorActivity { const E = episodes.length; // Range over the whole sample, and whether the dimension only takes a few values. const lo = new Float64Array(D).fill(Infinity); const hi = new Float64Array(D).fill(-Infinity); const uniques = Array.from({ length: D }, () => new Set()); for (const { actions, length: T } of episodes) { for (let t = 0; t < T; t++) { for (let d = 0; d < D; d++) { const v = actions[t * D + d]; if (v < lo[d]) lo[d] = v; if (v > hi[d]) hi[d] = v; if (uniques[d].size <= DISCRETE_THRESHOLD) uniques[d].add(v); } } } const motorRanges = new Float64Array(D); for (let d = 0; d < D; d++) motorRanges[d] = hi[d] - lo[d] || 1; // Is dimension d moving in episode e? p95(|Δa|) against 0.1% of its range. const active = new Uint8Array(E * D); for (let e = 0; e < E; e++) { const { actions, length: T } = episodes[e]; if (T < 2) continue; const absDeltas = new Float64Array(T - 1); for (let d = 0; d < D; d++) { for (let t = 1; t < T; t++) { absDeltas[t - 1] = Math.abs( actions[t * D + d] - actions[(t - 1) * D + d], ); } absDeltas.sort(); const p95 = absDeltas[Math.floor(absDeltas.length * 0.95)]; if (p95 >= motorRanges[d] * ACTIVITY_THRESHOLD) active[e * D + d] = 1; } } const globallyActive = new Uint8Array(D); for (let e = 0; e < E; e++) { for (let d = 0; d < D; d++) if (active[e * D + d]) globallyActive[d] = 1; } return { motorRanges, discrete: uniques.map((values) => values.size <= DISCRETE_THRESHOLD), active, globallyActive, }; } /** Velocity: the distribution of Δa per dimension, pooled over the episodes where it moves. */ function velocityStats( episodes: EpisodeSeries[], D: number, actionNames: string[], { motorRanges, discrete, active, globallyActive }: MotorActivity, ): AggVelocityStat[] { const E = episodes.length; const stats: AggVelocityStat[] = []; /** Calls `visit` with every Δa of dimension d that counts toward its statistics. */ const eachDelta = (d: number, visit: (delta: number) => void) => { for (let e = 0; e < E; e++) { // A dimension that never moves has no "active" episodes: describe all of them instead. if (globallyActive[d] && !active[e * D + d]) continue; const { actions, length: T } = episodes[e]; for (let t = 1; t < T; t++) { visit(actions[t * D + d] - actions[(t - 1) * D + d]); } } }; for (let d = 0; d < D; d++) { const motorRange = motorRanges[d]; const base = { name: shortName(actionNames[d]), motorRange, inactive: !globallyActive[d], discrete: discrete[d], }; let count = 0; let sum = 0; let maxAbs = 0; eachDelta(d, (v) => { count++; sum += v; const a = Math.abs(v); if (a > maxAbs) maxAbs = a; }); if (count === 0 || maxAbs === 0) { const bins = new Array(VELOCITY_BINS).fill(0); // Every change is zero: all of them in the middle bin. bins[(VELOCITY_BINS - 1) / 2] = count; stats.push({ ...base, std: 0, maxAbs: 0, bins, lo: 0, hi: 0 }); continue; } // The histogram's reach: the |Δa| that 99.5% of changes stay within, found from a fine // histogram rather than a sort (there can be millions of them). Up to the largest change, // one spike flattened everything else into the middle bins. const mean = sum / count; const fine = new Uint32Array(FINE_BINS); let varSum = 0; eachDelta(d, (v) => { varSum += (v - mean) ** 2; fine[ Math.min(FINE_BINS - 1, Math.floor((Math.abs(v) / maxAbs) * FINE_BINS)) ]++; }); let within = 0; let reach = maxAbs; for (let i = 0; i < FINE_BINS; i++) { within += fine[i]; if (within >= count * VELOCITY_REACH_QUANTILE) { reach = ((i + 1) / FINE_BINS) * maxAbs; break; } } // Symmetric about zero, with an odd number of bins, so "no change" is the middle one and the // two tails can be compared by eye. Changes beyond the reach land in the end bins. const bins = new Array(VELOCITY_BINS).fill(0); eachDelta(d, (v) => { const b = Math.floor(((v + reach) / (2 * reach)) * VELOCITY_BINS); bins[Math.max(0, Math.min(VELOCITY_BINS - 1, b))]++; }); stats.push({ ...base, std: Math.sqrt(varSum / count) / motorRange, maxAbs: maxAbs / motorRange, bins, lo: -reach / motorRange, hi: reach / motorRange, }); } return stats; } /** * The per-motor velocity statistics of one or more episodes. The cross-episode statistics include * them; this is for the episode on screen, before (or without) the dataset-wide analysis. */ export function computeVelocityStats( episodes: EpisodeSeries[], actionDim: number, actionNames: string[], ): AggVelocityStat[] { if (episodes.length === 0 || actionDim === 0) return []; return velocityStats( episodes, actionDim, actionNames, motorActivity(episodes, actionDim), ); } /** Everything but the lag statistics: one pass each over the frames. */ function baseStats(input: CrossEpisodeInput): { stats: BaseStats; activity: Activity; } { const { episodes, actionDim: D, actionNames } = input; const numTimeBins = input.numTimeBins ?? 50; const E = episodes.length; // ── Variance of each dimension across episodes, at matching points of episode progress ── const timeBins = Array.from( { length: numTimeBins }, (_, i) => i / (numTimeBins - 1), ); const sums = new Float64Array(numTimeBins * D); const sumsSq = new Float64Array(numTimeBins * D); for (const { actions, length: T } of episodes) { for (let b = 0; b < numTimeBins; b++) { const src = Math.min(Math.round(timeBins[b] * (T - 1)), T - 1) * D; const out = b * D; for (let d = 0; d < D; d++) { const v = actions[src + d]; sums[out + d] += v; sumsSq[out + d] += v * v; } } } const variance: number[][] = []; for (let b = 0; b < numTimeBins; b++) { const row: number[] = []; for (let d = 0; d < D; d++) { const mean = sums[b * D + d] / E; // Rounding can leave a constant dimension a hair below zero. row.push(Math.max(0, sumsSq[b * D + d] / E - mean * mean)); } variance.push(row); } // ── Movement per episode: mean L2 norm of the frame-to-frame action change ── const movementScores: LowMovementEpisode[] = episodes.map( ({ index, actions, length: T }) => { if (T < 2) return { episodeIndex: index, totalMovement: 0 }; let total = 0; for (let t = 1; t < T; t++) { let sumSq = 0; for (let d = 0; d < D; d++) { const delta = actions[t * D + d] - actions[(t - 1) * D + d]; sumSq += delta * delta; } total += Math.sqrt(sumSq); } return { episodeIndex: index, totalMovement: Math.round((total / (T - 1)) * 10000) / 10000, }; }, ); movementScores.sort((a, b) => a.totalMovement - b.totalMovement); const activity = motorActivity(episodes, D); const { motorRanges, active } = activity; const aggVelocity = velocityStats(episodes, D, actionNames, activity); // ── Jerkiness per episode: mean |Δa| over its moving dimensions, in units of motor range ── const jerkyEpisodes: JerkyEpisode[] = episodes .map(({ index, actions, length: T }, e) => { let sum = 0; let count = 0; for (let d = 0; d < D; d++) { if (!active[e * D + d]) continue; for (let t = 1; t < T; t++) { sum += Math.abs(actions[t * D + d] - actions[(t - 1) * D + d]) / motorRanges[d]; count++; } } return { episodeIndex: index, meanAbsDelta: count > 0 ? sum / count : 0 }; }) .sort((a, b) => b.meanAbsDelta - a.meanAbsDelta); return { stats: { actionNames, timeBins, variance, numEpisodes: E, lowMovementEpisodes: movementScores.slice(0, 10), aggVelocity, speedDistribution: movementScores.map((s) => ({ episodeIndex: s.episodeIndex, speed: s.totalMovement, })), jerkyEpisodes, }, activity: { active, windows: episodes.map(({ length }) => lagWindow(length)), }, }; } /** Autocorrelation per dimension, averaged over the episodes where it moves. */ function autocorrelation( input: CrossEpisodeInput, { active, windows }: Activity, ): AggAutocorrelation | null { const { episodes, actionDim: D, actionNames, fps } = input; const E = episodes.length; // The horizon follows the typical episode, not the shortest: one aborted recording used to // cut the chart short for the whole dataset. Episodes too short for it are left out. const spans = windows.map(([from, to]) => to - from).sort((a, b) => a - b); const maxLag = Math.min(MAX_ACF_LAG, Math.floor(median(spans) / 2)); if (maxLag < 2) return null; const acf = new Float64Array(D * maxLag); const used = new Uint32Array(D); for (let e = 0; e < E; e++) { const [from, to] = windows[e]; const n = to - from; if (n < maxLag * 2) continue; const { actions } = episodes[e]; const centered = new Float64Array(n); for (let d = 0; d < D; d++) { // A motor at rest is noise around a constant: it decorrelates at once, and would drag // both the average and the suggested chunk length down. if (!active[e * D + d]) continue; let mean = 0; for (let t = 0; t < n; t++) mean += actions[(from + t) * D + d]; mean /= n; let vari = 0; for (let t = 0; t < n; t++) { const c = actions[(from + t) * D + d] - mean; centered[t] = c; vari += c * c; } if (vari === 0) continue; for (let lag = 1; lag <= maxLag; lag++) { let s = 0; for (let t = 0; t < n - lag; t++) s += centered[t] * centered[t + lag]; acf[d * maxLag + lag - 1] += s / vari; } used[d]++; } } const dims: number[] = []; for (let d = 0; d < D; d++) if (used[d] > 0) dims.push(d); if (dims.length === 0) return null; for (const d of dims) { for (let l = 0; l < maxLag; l++) acf[d * maxLag + l] /= used[d]; } const shortKeys = dims.map((d) => shortName(actionNames[d])); const chartData = Array.from({ length: maxLag }, (_, lag) => { const row: Record = { lag: lag + 1, time: (lag + 1) / fps, }; dims.forEach((d, i) => { row[shortKeys[i]] = acf[d * maxLag + lag]; }); return row; }); // Suggested chunk: the median, over dimensions, of the first lag where the ACF is below 0.5. const crossings: number[] = []; for (const d of dims) { for (let l = 0; l < maxLag; l++) { if (acf[d * maxLag + l] < 0.5) { crossings.push(l + 1); break; } } } crossings.sort((a, b) => a - b); return { chartData, suggestedChunk: crossings.length > 0 ? median(crossings) : null, shortKeys, }; } /** State–action alignment: cross-correlation of Δaction(t) and Δstate(t + lag). */ function alignment( input: CrossEpisodeInput, { windows }: Activity, ): AggAlignment | null { const { episodes, actionDim: D, stateDim, actionNames, stateNames } = input; const E = episodes.length; if (stateDim === 0) return null; // Pair each action with the state of the same name, else by position. const pairs: [number, number][] = []; for (let ai = 0; ai < D; ai++) { const si = stateNames.indexOf(shortName(actionNames[ai])); if (si >= 0) pairs.push([ai, si]); } if (pairs.length === 0) { const count = Math.min(D, stateDim); for (let i = 0; i < count; i++) pairs.push([i, i]); } if (pairs.length === 0) return null; const maxLag = ALIGNMENT_MAX_LAG; const numLags = 2 * maxLag + 1; const corrSums = new Float64Array(pairs.length * numLags); const corrCounts = new Uint32Array(pairs.length * numLags); for (let e = 0; e < E; e++) { const { actions, states } = episodes[e]; if (!states) continue; const [from, to] = windows[e]; const n = to - from; if (n < 10) continue; const effN = n - 1; // Centered changes, and running sums of their squares: a lag's variances are then two // subtractions, which leaves one multiply-add per frame in the loop that dominates. const aCentered = new Float64Array(effN); const sCentered = new Float64Array(effN); const aSquares = new Float64Array(effN + 1); const sSquares = new Float64Array(effN + 1); for (let p = 0; p < pairs.length; p++) { const [ai, si] = pairs[p]; let aMean = 0; let sMean = 0; for (let t = 0; t < effN; t++) { const row = from + t; aCentered[t] = actions[(row + 1) * D + ai] - actions[row * D + ai]; sCentered[t] = states[(row + 1) * stateDim + si] - states[row * stateDim + si]; aMean += aCentered[t]; sMean += sCentered[t]; } aMean /= effN; sMean /= effN; for (let t = 0; t < effN; t++) { aCentered[t] -= aMean; sCentered[t] -= sMean; aSquares[t + 1] = aSquares[t] + aCentered[t] * aCentered[t]; sSquares[t + 1] = sSquares[t] + sCentered[t] * sCentered[t]; } for (let li = 0; li < numLags; li++) { const lag = li - maxLag; // Frames t where both Δaction(t) and Δstate(t + lag) exist. const tFrom = Math.max(0, -lag); const tTo = Math.min(effN, effN - lag); if (tTo <= tFrom) continue; let sum = 0; for (let t = tFrom; t < tTo; t++) { sum += aCentered[t] * sCentered[t + lag]; } const aVar = aSquares[tTo] - aSquares[tFrom]; const sVar = sSquares[tTo + lag] - sSquares[tFrom + lag]; const denom = Math.sqrt(aVar * sVar); if (denom > 0) { corrSums[p * numLags + li] += sum / denom; corrCounts[p * numLags + li]++; } } } } // A pair that never moved (a motor at rest in every episode) has no correlation to report; // counted as a flat zero it would pull the mean and the minimum toward zero. const avgCorrs: number[][] = []; for (let p = 0; p < pairs.length; p++) { const row: number[] = []; let any = false; for (let li = 0; li < numLags; li++) { const c = corrCounts[p * numLags + li]; if (c > 0) any = true; row.push(c > 0 ? corrSums[p * numLags + li] / c : 0); } if (any) avgCorrs.push(row); } if (avgCorrs.length === 0) return null; const ccData = Array.from({ length: numLags }, (_, li) => { const vals = avgCorrs.map((pc) => pc[li]); return { lag: li - maxLag, max: Math.max(...vals), mean: vals.reduce((a, b) => a + b, 0) / vals.length, min: Math.min(...vals), }; }); const peak = (key: "max" | "mean" | "min") => { let best = ccData[0]; for (const row of ccData) if (row[key] > best[key]) best = row; return { lag: best.lag, corr: best[key] }; }; const maxPeak = peak("max"); const meanPeak = peak("mean"); const minPeak = peak("min"); const perPairPeakLags = avgCorrs.map((pc) => { let bestLag = 0; let best = -Infinity; for (let li = 0; li < pc.length; li++) { if (pc[li] > best) { best = pc[li]; bestLag = li - maxLag; } } return bestLag; }); return { ccData, meanPeakLag: meanPeak.lag, meanPeakCorr: meanPeak.corr, maxPeakLag: maxPeak.lag, maxPeakCorr: maxPeak.corr, minPeakLag: minPeak.lag, minPeakCorr: minPeak.corr, lagRangeMin: Math.min(...perPairPeakLags), lagRangeMax: Math.max(...perPairPeakLags), numPairs: avgCorrs.length, }; } export function computeCrossEpisodeStats( input: CrossEpisodeInput, ): CrossEpisodeVarianceData | null { if (input.episodes.length < 2 || input.actionDim === 0) return null; const { stats, activity } = baseStats(input); return { ...stats, aggAutocorrelation: autocorrelation(input, activity), aggAlignment: alignment(input, activity), }; } /** * The same statistics with a pause between the three stages. Each is a tight loop of tens to * hundreds of milliseconds; run back to back on the main thread they freeze the page for their sum. */ export async function computeCrossEpisodeStatsInSteps( input: CrossEpisodeInput, pause: () => Promise, ): Promise { if (input.episodes.length < 2 || input.actionDim === 0) return null; const { stats, activity } = baseStats(input); await pause(); const aggAutocorrelation = autocorrelation(input, activity); await pause(); return { ...stats, aggAutocorrelation, aggAlignment: alignment(input, activity), }; }