Hugging Face's logo

Spaces:
huggingface-mic
/
mic-coverage
Running

App Files Community
mic-coverage / index.html
tarekziade's picture
tarekziade HF Staff
Say how each fleet runs in CI
ce35c68 verified
Raw History Blame Contribute Delete
16.4 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>MIC coverage</title>
<style>
:root{--bg:#fff;--fg:#1f2328;--muted:#656d76;--line:#d0d7de;--soft:#f6f8fa;--accent:#0969da;--done:#1a7f37;--reach:#bf8700;--no:#cf222e;--goal:#8250df;--goal-bg:#fbefff;--none:#8c959f}
@media (prefers-color-scheme:dark){:root{--bg:#0d1117;--fg:#e6edf3;--muted:#8d96a0;--line:#30363d;--soft:#161b22;--accent:#4493f8;--done:#3fb950;--reach:#d29922;--no:#f85149;--goal:#ab7df8;--goal-bg:#2a1f3d;--none:#6e7681}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.45 system-ui,-apple-system,"Segoe UI",sans-serif}
main{max-width:1200px;margin:0 auto;padding:24px 16px 48px}
h1{font-size:22px;margin:0 0 6px}
.lead{color:var(--muted);margin:0 0 18px;max-width:85ch}
.bars{display:grid;gap:14px;margin-bottom:18px}
.bar-head{display:flex;justify-content:space-between;gap:12px;flex-wrap:wrap;margin-bottom:5px}
.bar-head strong{font-size:14px}.bar-head span{color:var(--muted);font-size:13px}
.bar{position:relative;display:flex;height:22px;border-radius:6px;overflow:hidden;background:var(--soft);border:1px solid var(--line)}
.bar i{display:block;height:100%;transition:flex-grow .3s}
.bar .Done{background:var(--done)}.bar .Reachable{background:var(--reach)}.bar .Unreachable{background:var(--no);opacity:.55}.bar .NoCheckpoint{background:var(--none)}
.bar .mark{position:absolute;top:-3px;bottom:-3px;width:2px;background:var(--fg)}
.legend{display:flex;gap:16px;flex-wrap:wrap;color:var(--muted);font-size:12px;margin-top:5px}
.legend b{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:5px;vertical-align:-1px}
.stats{display:flex;gap:10px;flex-wrap:wrap;margin-bottom:14px}
.stat{border:1px solid var(--line);border-radius:6px;padding:8px 12px;background:var(--soft);cursor:pointer;font:inherit;color:inherit;text-align:left}
.stat strong{display:block;font-size:20px}
.stat[aria-pressed=true]{outline:2px solid var(--accent)}
.stat.goal strong{color:var(--goal)}
.controls{display:flex;gap:10px;align-items:center;margin-bottom:10px;flex-wrap:wrap}
input[type=search]{flex:1;min-width:200px;max-width:360px;padding:6px 10px;border:1px solid var(--line);border-radius:6px;background:var(--bg);color:var(--fg);font:inherit}
.count{color:var(--muted)}
.wrap{overflow-x:auto;border:1px solid var(--line);border-radius:6px}
table{width:100%;border-collapse:collapse}
th,td{padding:6px 10px;border-bottom:1px solid var(--line);text-align:left;white-space:nowrap}
th{position:sticky;top:0;background:var(--soft);cursor:pointer;user-select:none;font-weight:600}
th:hover{color:var(--accent)}
th .arrow{color:var(--muted);font-size:11px;margin-left:4px}
td.num,th.num{text-align:right;font-variant-numeric:tabular-nums}
tbody tr:hover{background:var(--soft)}
tr.goal td:first-child{box-shadow:inset 3px 0 0 var(--goal)}
tr.goal.miss{background:var(--goal-bg)}
.badge{display:inline-block;margin-left:6px;padding:0 6px;border-radius:10px;font-size:11px;font-weight:600;color:var(--goal);border:1px solid var(--goal);vertical-align:1px}
td.top{white-space:normal;overflow-wrap:anywhere;min-width:12ch}
td.gated{white-space:nowrap;font-size:12px}td.why{white-space:normal;overflow-wrap:anywhere;min-width:14ch;max-width:36ch}
@media (max-width:700px){main{padding:16px 12px 40px}.wrap{border:0;overflow:visible}table,tbody,tr,td{display:block}thead tr{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:8px}thead th{position:static;border:1px solid var(--line);border-radius:12px;padding:2px 8px;font-size:12px}tbody tr{border:1px solid var(--line);border-radius:6px;margin-bottom:8px;padding:6px 10px}td{border:0;padding:1px 0;white-space:normal;overflow-wrap:anywhere;text-align:left!important}td:empty{display:none}td::before{content:attr(data-label);display:inline-block;min-width:11em;color:var(--muted);font-size:12px}td.why,td.top{min-width:0;max-width:none}tr.goal td:first-child{box-shadow:none}tr.goal{border-left:3px solid var(--goal)}}.legend span{min-width:0}
a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}
.state{font-weight:600}td.Done{color:var(--done)}td.Reachable{color:var(--reach)}td.Unreachable{color:var(--no)}td.NoCheckpoint{color:var(--muted)}
.foot{color:var(--muted);font-size:12px;margin-top:14px}
.mic{display:block;margin:2px 0}.ci{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:6px;vertical-align:1px;background:var(--done)}.ci.warn{background:var(--reach)}.ci.bad{background:var(--no)}a.ci:hover{outline:2px solid var(--accent)}td.mics{white-space:normal;min-width:24ch;max-width:34ch}.mic a{overflow-wrap:anywhere}.fleet{display:inline-block;margin-right:6px;padding:0 5px;border-radius:4px;font-size:10px;font-weight:600;letter-spacing:.04em;text-transform:uppercase;border:1px solid var(--line);color:var(--muted);vertical-align:1px}.fleet.control{color:var(--fg);background:var(--soft)}.src{display:block;color:var(--muted);font-size:12px;overflow-wrap:anywhere}.note{color:var(--muted);font-size:12px;margin:0 0 8px}
</style>
</head>
<body>
<main>
<h1>MIC coverage of Transformers model types</h1>
<p class="lead" id="lead">Loading…</p>
<div class="bars" id="bars"></div>
<div class="stats" id="stats"></div>
<div class="controls">
<input type="search" id="q" placeholder="Filter by model type, reason, checkpoint or MIC" aria-label="Filter rows">
<span class="count" id="count"></span>
</div>
<p class="note">MIC column: each link shows how the fixture was shrunk. A model type with several MICs tags them: <span class="fleet control">control</span>an original catalog contract from the transformers-mic repo; Transformers PR CI runs it in float32, and the nightly also checks it on the real checkpoint; <span class="fleet">prototype</span>one drafted by tmic's planner and reviewed; PR CI runs it in bfloat16; when they come from different checkpoints, the source is shown under the link. The dot is the MIC on Transformers <code>main</code>: <span class="ci"></span>no failure in 30 days, <span class="ci warn"></span>passing now after failures, <span class="ci bad"></span>failing now. Details on the <a href="https://transformers-ci.huggingface.cool/d/mic-failures/mic-failures?from=now-4w&amp;to=now&amp;timezone=browser" target="_blank" rel="noopener">MIC failures dashboard</a>.</p>
<div class="wrap"><table>
<thead><tr>
<th data-col="5" scope="col" class="num" title="Rank by downloads">#<span class="arrow"></span></th>
<th data-col="0" scope="col">Model type<span class="arrow"></span></th>
<th data-col="1" scope="col">State<span class="arrow"></span></th>
<th data-col="2" scope="col">Why not reachable<span class="arrow"></span></th>
<th data-col="3" scope="col" class="num">Downloads (30 days)<span class="arrow"></span></th>
<th data-col="4" scope="col">Top checkpoint<span class="arrow"></span></th>
<th scope="col" title="Source checkpoint gated on the Hub (auto or manual approval)">Gated</th>
<th scope="col" title="The MIC fixture, linking to how it was shrunk (tmic shrink)">MIC (shrink)</th>
</tr></thead>
<tbody id="rows"></tbody>
</table></div>
<p class="foot" id="foot"></p>
</main>
<script>
(async () => {
const {rows, meta, shrink = {}, gated = {}, results = {}} = await (await fetch('data.json')).json();
const ciMeta = meta.results;
const STATES = ['Done', 'Reachable', 'Unreachable', 'No checkpoint'];
const cls = s => s.replace(' ', '');
const ORDER = {Done: 0, Reachable: 1, Unreachable: 2, 'No checkpoint': 3};
const GOAL_SHARE = 0.3;
const total = rows.length, goalSize = Math.ceil(total * GOAL_SHARE);
// Rank by downloads (ties by name); the top 30% are the goal.
[...rows].sort((a, b) => b[3] - a[3] || a[0].localeCompare(b[0])).forEach((r, i) => { r[5] = i + 1; r[6] = i < goalSize; });
const tally = list => { const c = {Done: 0, Reachable: 0, Unreachable: 0, 'No checkpoint': 0}; list.forEach(r => c[r[1]]++); return c; };
const all = tally(rows), goal = tally(rows.filter(r => r[6]));
const pct = (n, of) => (100 * n / of).toFixed(1) + '%';
const covered = c => c.Done + c.Reachable;
document.getElementById('lead').textContent =
`${all.Done} of ${total} model types have a MIC contract; ${covered(all)} (${pct(covered(all), total)}) are done or reachable ` +
`with the generic shrink rule. Goal: cover the ${goalSize} most downloaded model types (the top ${GOAL_SHARE * 100}%), ` +
`marked in purple below. ${all['No checkpoint']} model types have no checkpoint on the Hub and cannot be treated. Click a column to sort; click again to reverse.`;
const bars = document.getElementById('bars');
function bar(title, counts, of, target) {
const box = document.createElement('div');
const head = document.createElement('div'); head.className = 'bar-head';
const t = document.createElement('strong'); t.textContent = title;
const s = document.createElement('span');
s.textContent = `${counts.Done} done · ${counts.Reachable} reachable · ${counts.Unreachable} unreachable · ${counts['No checkpoint']} no checkpoint — ${pct(covered(counts), of)} done or reachable`;
head.append(t, s);
const b = document.createElement('div'); b.className = 'bar'; b.setAttribute('role', 'img');
b.setAttribute('aria-label', `${title}: ${s.textContent}`);
for (const k of STATES) {
const seg = document.createElement('i'); seg.className = cls(k); seg.style.flexGrow = counts[k];
seg.title = `${k}: ${counts[k]} (${pct(counts[k], of)})`;
b.appendChild(seg);
}
if (target) {
const m = document.createElement('span'); m.className = 'mark'; m.style.left = `calc(${target * 100}% - 1px)`;
m.title = `${target * 100}% target`; b.appendChild(m);
}
box.append(head, b);
bars.appendChild(box);
}
bar(`Goal: top ${GOAL_SHARE * 100}% most downloaded (${goalSize} model types)`, goal, goalSize);
bar(`All ${total} model types`, all, total);
const legend = document.createElement('div'); legend.className = 'legend';
legend.innerHTML = '<span><b style="background:var(--done)"></b>Done: has a MIC contract</span>' +
'<span><b style="background:var(--reach)"></b>Reachable: generic shrink rule builds and runs it</span>' +
'<span><b style="background:var(--no);opacity:.55"></b>Unreachable</span>' +
'<span><b style="background:var(--none)"></b>No checkpoint: no Hub checkpoint has this model type, so it cannot be treated</span>';
bars.appendChild(legend);
let sort = {col: 3, dir: -1}, only = '';
const stats = document.getElementById('stats');
const filters = [['', 'All model types', total], ['goal', `Top ${GOAL_SHARE * 100}% (goal)`, goalSize], ...STATES.map(s => [s, s, all[s]])];
for (const [v, label, n] of filters) {
const b = document.createElement('button');
b.className = 'stat' + (v === 'goal' ? ' goal' : ''); b.type = 'button'; b.dataset.v = v;
b.innerHTML = `<strong>${n}</strong>${label}`;
b.onclick = () => { only = v; render(); };
stats.appendChild(b);
}
const cmp = (a, b) => {
const c = sort.col;
let d;
if (c === 3 || c === 5) d = a[c] - b[c];
else if (c === 1) d = ORDER[a[1]] - ORDER[b[1]] || b[3] - a[3];
else d = String(a[c]).localeCompare(String(b[c]), undefined, {sensitivity: 'base'});
return d * sort.dir || a[0].localeCompare(b[0]);
};
document.querySelectorAll('th[data-col]').forEach(th => th.onclick = () => {
const col = +th.dataset.col;
sort = sort.col === col ? {col, dir: -sort.dir} : {col, dir: col === 3 ? -1 : 1};
render();
});
const LABELS = [...document.querySelectorAll('thead th')].map(th => th.firstChild.textContent.trim());
const q = document.getElementById('q');
q.oninput = render;
const keep = r => !only || (only === 'goal' ? r[6] : r[1] === only);
function render() {
const f = q.value.trim().toLowerCase();
const shown = rows.filter(r => keep(r) && (!f || [...r.slice(0, 5), ...(shrink[r[0]] || []).map(e => e[0])].join(' ').toLowerCase().includes(f))).sort(cmp);
document.getElementById('rows').replaceChildren(...shown.map(r => {
const tr = document.createElement('tr');
if (r[6]) tr.className = 'goal' + (r[1] === 'Unreachable' ? ' miss' : '');
const td = (text, cls) => { const c = document.createElement('td'); if (cls) c.className = cls; c.textContent = text; tr.appendChild(c); return c; };
td(r[5], 'num');
const name = td(r[0]);
if (r[6]) { const b = document.createElement('span'); b.className = 'badge'; b.textContent = `Top ${GOAL_SHARE * 100}%`; name.appendChild(b); }
td(r[1], `state ${cls(r[1])}`);
td(r[2], 'why');
td(r[3].toLocaleString('en-US'), 'num');
const top = td('', 'top');
if (r[4]) {
const a = document.createElement('a');
a.href = `https://huggingface.co/${r[4]}`; a.target = '_blank'; a.rel = 'noopener'; a.textContent = r[4];
top.appendChild(a);
}
td(gated[r[0]] ? `🔒 ${gated[r[0]]}` : '', 'gated');
const view = td('', 'mics');
const entries = (r[1] === 'Done' ? shrink[r[0]] || [] : []).slice().sort((a, b) => (a[2] === 'control' ? 0 : 1) - (b[2] === 'control' ? 0 : 1));
const several = entries.length > 1, mixed = new Set(entries.map(e => e[3])).size > 1;
for (const [page, findings, fleet, source] of entries) {
const line = document.createElement('span'); line.className = 'mic';
if (several && fleet) {
const tag = document.createElement('span');
tag.className = 'fleet' + (fleet === 'control' ? ' control' : '');
tag.textContent = fleet === 'control' ? 'control' : 'prototype';
line.appendChild(tag);
}
if (ciMeta) {
const [now, fails] = results[page] || [0, 0];
const dot = document.createElement('a');
dot.className = 'ci' + (now ? ' bad' : fails ? ' warn' : '');
dot.href = `https://transformers-ci.huggingface.cool/mic?q=${encodeURIComponent(page)}&days=${ciMeta.days}&sticky_only=false`;
dot.target = '_blank'; dot.rel = 'noopener';
dot.title = now ? `Failing now on main (${fails} failed checks in ${ciMeta.days} days)` : fails ? `Passing now; ${fails} failed checks in ${ciMeta.days} days` : `No failure on main in ${ciMeta.days} days`;
dot.setAttribute('aria-label', dot.title);
line.appendChild(dot);
}
const a = document.createElement('a');
a.href = `shrink/${page}.html`;
a.textContent = page.replace(/-mic$/, '') + (findings ? ` ⚠ ${findings}` : '');
a.title = `${page}${source ? ` from ${source}` : ''}: ${findings ? findings + ' finding(s) to review' : 'same architecture, smaller'}`;
line.appendChild(a);
if (mixed && source) { const s = document.createElement('span'); s.className = 'src'; s.textContent = `from ${source.split('/').pop()}`; s.title = source; line.appendChild(s); }
view.appendChild(line);
}
tr.querySelectorAll('td').forEach((c, i) => c.dataset.label = LABELS[i]);
return tr;
}));
document.querySelectorAll('th[data-col]').forEach(th => {
const on = +th.dataset.col === sort.col;
th.querySelector('.arrow').textContent = on ? (sort.dir > 0 ? '▲' : '▼') : '';
th.setAttribute('aria-sort', on ? (sort.dir > 0 ? 'ascending' : 'descending') : 'none');
});
stats.querySelectorAll('.stat').forEach(b => b.setAttribute('aria-pressed', String(b.dataset.v === only)));
document.getElementById('count').textContent = `${shown.length} of ${total} shown`;
}
const short = s => s.slice(0, 7);
document.getElementById('foot').innerHTML =
`Snapshot ${meta.generated}. Downloads = Hub 30-day count summed over each type's top 1,000 checkpoints whose ` +
`<code>config.model_type</code> matches. Sources: ` +
`<a href="https://huggingface.co/datasets/huggingface-mic/transformers-fleet/tree/${meta.fleet}">MIC fleet @${short(meta.fleet)}</a>, ` +
`<a href="https://github.com/huggingface/transformers/blob/${meta.transformers}/src/transformers/models/auto/auto_mappings.py">Transformers model types @${short(meta.transformers)}</a>, ` +
`<a href="https://github.com/huggingface/transformers-mic/blob/${meta.sweep}/experiments/results/sweep.json">shrink sweep @${short(meta.sweep)}</a>.`;
render();
})();
</script>
</body>
</html>