Say why a listed model was not tested

app.js

@@ -12,6 +12,9 @@ const ORDERS = ["selected", "trending", "downloads"];
 const HUB_SORTS = { trending: "trendingScore", downloads: "downloads" };
 // How many of the Hub's top models a trending or downloads list shows.
 const TOP = 100;
+// The test instance: 4 Trainium2 devices of 96 GiB of HBM each.
+const DEVICES = 4;
+const DEVICE_HBM = 96 * 1024 ** 3;
 const COLUMNS = ["model_id", "tp", "result", "wall (s)", "Memory (GB)", "notes"];
 
 // An element with attributes and children. Strings become text nodes, never HTML.
@@ -41,19 +44,37 @@ function cached(key, load) {
   return cache.get(key);
 }
 
+// The rows of a CSV file as objects keyed by its header. Fields may be quoted.
+function parseCsv(text) {
+  const rows = [[]];
+  let field = "";
+  let quoted = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (quoted && c === '"' && text[i + 1] === '"') field += text[i++];
+    else if (c === '"') quoted = !quoted;
+    else if (quoted || (c !== "," && c !== "\n" && c !== "\r")) field += c;
+    else if (c === ",") { rows.at(-1).push(field); field = ""; }
+    else if (c === "\n") { rows.at(-1).push(field); field = ""; rows.push([]); }
+  }
+  if (field || rows.at(-1).length) rows.at(-1).push(field);
+  const [header, ...lines] = rows.filter((row) => row.length);
+  return lines.map((line) => Object.fromEntries(header.map((key, i) => [key, line[i]])));
+}
+
 // The dataset's files at its latest commit: one per (neuronx-cc, library version)
-// pair, and one selection per library.
+// pair, and the selection and excluded list of each library.
 async function loadIndex() {
   const info = await (await fetchOk(`${HUB}/api/datasets/${DATASET}`)).json();
   const pairs = [];
-  const selections = new Map();
+  const lists = { selection: new Map(), excluded: new Map() };
   for (const { rfilename: path } of info.siblings) {
     const pair = path.match(/^data\/neuronxcc-([^/]+)\/(.+)-([^-/]+)\.parquet$/);
     if (pair) pairs.push({ sdk: pair[1], library: pair[2], version: pair[3], path });
-    const selection = path.match(/^selection\/(.+)\.csv$/);
-    if (selection) selections.set(selection[1], path);
+    const list = path.match(/^(selection|excluded)\/(.+)\.csv$/);
+    if (list) lists[list[1]].set(list[2], path);
   }
-  return { sha: info.sha, pairs, selections };
+  return { sha: info.sha, pairs, ...lists };
 }
 
 const index = await loadIndex().catch((e) => e);
@@ -63,11 +84,12 @@ const loadRuns = (path) => cached(path, async () =>
   parquetReadObjects({ file: await (await fetchOk(fileUrl(path))).arrayBuffer() }));
 
 // Each model's position in a selection, best-ranked first.
-const loadSelection = (path) => cached(path, async () => {
-  const [header, ...lines] = (await (await fetchOk(fileUrl(path))).text()).trim().split("\n");
-  const column = header.split(",").indexOf("model_id");
-  return new Map(lines.map((line, rank) => [line.split(",")[column], rank]));
-});
+const loadSelection = (path) => cached(path, async () =>
+  new Map(parseCsv(await (await fetchOk(fileUrl(path))).text()).map((row, rank) => [row.model_id, rank])));
+
+// Each left-out model's row of an excluded list.
+const loadExcluded = (path) => cached(path, async () =>
+  new Map(parseCsv(await (await fetchOk(fileUrl(path))).text()).map((row) => [row.model_id, row])));
 
 // A model's task on the Hub, or "unknown" if the Hub no longer has it.
 const loadTask = (id) => cached(`task:${id}`, async () => {
@@ -78,14 +100,15 @@ const loadTask = (id) => cached(`task:${id}`, async () => {
   return (await response.json()).pipeline_tag ?? "unknown";
 }).catch(() => "unknown");
 
-// The Hub's top models of a library and task, as its model list shows them.
+// The Hub's top models of a library and task, as its model list shows them,
+// with their parameter count when the Hub knows it.
 const loadTop = (library, task, order) => cached(`top:${library}:${task}:${order}`, async () => {
   const sort = HUB_SORTS[order];
-  const query = new URLSearchParams({
-    library, pipeline_tag: task, sort, direction: "-1", limit: TOP, "expand[]": sort,
-  });
+  const query = new URLSearchParams({ library, pipeline_tag: task, sort, direction: "-1", limit: TOP });
+  query.append("expand[]", sort);
+  query.append("expand[]", "safetensors");
   const models = await (await fetchOk(`${HUB}/api/models?${query}`)).json();
-  return models.map((model) => ({ id: model.id, score: model[sort] ?? 0 }));
+  return models.map((model) => ({ id: model.id, score: model[sort] ?? 0, params: model.safetensors?.total }));
 });
 
 // Newest first. A version that is not a release, such as a git commit pin,
@@ -154,12 +177,13 @@ async function renderPanel() {
     tasks,
     results);
 
-  let runs, rank, taskOf;
+  let runs, rank, exclusions, taskOf;
   try {
-    const selection = index.selections.get(active);
-    [runs, rank] = await Promise.all([
+    const [selection, excluded] = [index.selection.get(active), index.excluded.get(active)];
+    [runs, rank, exclusions] = await Promise.all([
       loadRuns(pairs.find((p) => p.sdk === s.sdk && p.version === s.version).path),
       selection ? loadSelection(selection) : new Map(),
+      excluded ? loadExcluded(excluded) : new Map(),
     ]);
     taskOf = new Map(await Promise.all(runs.map(async (run) => [run.model_id, await loadTask(run.model_id)])));
   } catch (e) {
@@ -178,13 +202,23 @@ async function renderPanel() {
     if (s.order === "selected") {
       return runs.filter((run) => rank.has(run.model_id) && ticked.includes(taskOf.get(run.model_id)))
         .sort((a, b) => rank.get(a.model_id) - rank.get(b.model_id) || a.tp - b.tp)
-        .map((run) => [run.model_id, run]);
+        .map((run) => ({ id: run.model_id, run }));
     }
     const lists = await Promise.all(ticked.map((t) => loadTop(active, t, s.order)));
     return [...new Map(lists.flat().map((model) => [model.id, model])).values()]
       .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id))
       .slice(0, TOP)
-      .flatMap(({ id }) => (runsOf.get(id) ?? [null]).map((run) => [id, run]));
+      .flatMap((model) => (runsOf.get(model.id) ?? [null]).map((run) => ({ ...model, run })));
+  }
+
+  // Why a model was not tested: weights larger than the test instance's HBM, or
+  // the reason the selection left it out, with the value its rule found.
+  function untestedReason({ id, params }) {
+    if (params * 2 > DEVICES * DEVICE_HBM) return `Requires more than ${DEVICES} Trainium2 devices`;
+    const exclusion = exclusions.get(id);
+    if (!exclusion) return "";
+    return exclusion.matched_pattern.includes("=") ? `${exclusion.reason} (${exclusion.matched_pattern})`
+      : exclusion.reason;
   }
 
   const renderResults = async () => {
@@ -204,7 +238,7 @@ async function renderPanel() {
           : "The Hub lists no model for the ticked tasks."));
       return;
     }
-    const tested = shown.map(([, run]) => run).filter(Boolean);
+    const tested = shown.map((entry) => entry.run).filter(Boolean);
     const counts = new Map();
     for (const run of tested) counts.set(run.result, (counts.get(run.result) ?? 0) + 1);
     const failures = [...counts].filter(([result]) => result !== "PASS").sort(([a], [b]) => a.localeCompare(b));
@@ -217,20 +251,20 @@ async function renderPanel() {
         : `None of these ${untested} models was tested on this pair.`),
       h("div", { class: "scroll" }, h("table", {},
         h("thead", {}, h("tr", {}, COLUMNS.map((c) => h("th", {}, c)))),
-        h("tbody", {}, shown.map(([id, run]) => run ? h("tr", {
+        h("tbody", {}, shown.map(({ run, ...model }) => run ? h("tr", {
           tabindex: 0,
           onclick: () => showRun(run),
           onkeydown: (e) => { if (e.key === "Enter") showRun(run); },
         },
-        h("td", {}, h("code", {}, id)),
+        h("td", {}, h("code", {}, model.id)),
         h("td", { class: "num" }, run.tp),
         h("td", { class: "result" }, badge(run.result)),
         h("td", { class: "num" }, run.wall_s ?? ""),
         h("td", { class: "num" }, memory(run)),
         h("td", { class: "notes" }, run.notes ?? ""))
           : h("tr", { class: "untested" },
-            h("td", {}, h("code", {}, id)), h("td"), h("td", { class: "result" }, "not tested"),
-            h("td"), h("td"), h("td")))))));
+            h("td", {}, h("code", {}, model.id)), h("td"), h("td", { class: "result" }, "not tested"),
+            h("td"), h("td"), h("td", { class: "notes" }, untestedReason(model))))))));
   };
   tasks.append(...taskList.map((t) => h("label", {},
     h("input", {

index.html

@@ -105,9 +105,10 @@ summary { cursor: pointer; color: var(--accent); margin: 12px 0 4px; }
 <h1>torch-neuronx model support</h1>
 <p class="intro">
   Which Hugging Face Hub models run unmodified on AWS Trainium through the native PyTorch backend
-  <a href="https://github.com/aws-neuron/torch-neuronx">torch-neuronx</a>. Select a model to see how
-  to run it. The <a href="https://huggingface.co/datasets/aws-neuron/torch-neuronx-model-support">dataset
-  card</a> describes each test and what each result means.
+  <a href="https://github.com/aws-neuron/torch-neuronx">torch-neuronx</a>. For now, only models that
+  fit on 4 Trainium2 devices (TP=16) are tested. Select a model to see how to run it. The
+  <a href="https://huggingface.co/datasets/aws-neuron/torch-neuronx-model-support">dataset card</a>
+  describes each test and what each result means.
 </p>
 <nav id="tabs" role="tablist"></nav>
 <main id="panel"><p class="status">Loading…</p></main>