Blame for instruction-audit.js

/* Current RLE catalog comparison, separate from recorded evaluation evidence. */
window.InstructionAudit = (() => {
  const cache = new Map();
  const esc = value => String(value ?? '').replace(/[&<>"']/g, c => ({'&':'&amp;', '<':'&lt;', '>':'&gt;', '"':'&quot;', "'":'&#39;'}[c]));
  const link = (url, label) => `<a href="${esc(url)}" target="_blank" rel="noopener">${esc(label)} ↗</a>`;
  const poolNames = {seen:'Seen templates', unseen:'Unseen templates', r1_distraction:'R1 · Distraction', r2_commonsense:'R2 · Commonsense', r3_reasoning:'R3 · Reasoning'};

  async function render({panel, root, data, measuredInstructions, diff, isCurrent}) {
    const main = document.querySelector('#main');
    document.title = `Instructions audit · ${panel[0].toUpperCase() + panel.slice(1)} — Programmable VLA`;
    main.innerHTML = '<div class="loading"><span class="spinner"></span>Loading instruction comparison…</div>';
    try {
      if (!cache.has(panel)) cache.set(panel, fetch(root + `instruction-audit/${panel}.json`).then(response => {
        if (!response.ok) throw Error(`Instruction snapshot unavailable (${response.status})`);
        return response.json();
      }).catch(error => {cache.delete(panel); throw error;}));
      const audit = await cache.get(panel);
      if (!isCurrent()) return;
      mount({main, panel, root, data, measuredInstructions, diff, audit});
    } catch (error) {
      if (isCurrent()) main.innerHTML = `<div class="empty-state"><h1>Instruction audit unavailable</h1><p>${esc(error.message)}</p><a href="${esc(root + `instruction-audit/${panel}.json`)}">Open the audit snapshot</a></div>`;
    }
  }

  function mount({main, panel, root, data, measuredInstructions, diff, audit}) {
    const filters = {query:'', family:'', status:'', binding:''};
    const sourceRoot = root + 'instruction-audit/';
    const variants = new Map(data.variants.map(value => [value.id, value.label || value.agent]));
    const episodes = new Map();
    for (const episode of data.episodes) {
      if (!episodes.has(episode.task_key)) episodes.set(episode.task_key, []);
      episodes.get(episode.task_key).push(episode);
    }
    const rows = audit.tasks.map((task, index) => ({...task, index, difference:diff(task.native, task.measured)}));
    const changed = rows.filter(task => task.status === 'changed').length;
    const countBindings = rows.filter(task => Object.keys(task.bindings).length).length;
    const scopeName = panel[0].toUpperCase() + panel.slice(1);
    const sections = [['easy', 'Easy', root + 'index.html#instructions'], ['hard', 'Hard', root + 'hard/index.html#instructions'], ['plus', 'Plus', root + 'plus/index.html#instructions']];
    main.innerHTML = `<section class="audit-intro"><div class="eyebrow">${esc(scopeName)} · Current RLE catalog</div><h1>Instructions audit</h1><p>Current RLE instruction on the left. Pinned native task description on the right.</p>
      <nav class="audit-suite-links" aria-label="Instruction audit suites">${sections.map(([key, name, href]) => `<a href="${esc(href)}" ${key === panel ? 'aria-current="page"' : ''}>${name}</a>`).join('')}</nav>
      <p class="audit-selection-note">${esc(audit.comparison_basis)} Curly-braced RLE fields are filled from the initialized scene before the first model call.</p>
      <aside class="audit-history-note"><strong>Catalog comparison, separate from published results</strong><p>${esc(audit.historical_note)} Native success predicates are unchanged; clearer language does not add missing checks to a verifier.</p>${panel === 'plus' ? '<p>Language-plus uses the same explicit RLE goal as the other configurations. This removes the original R1/R2/R3 language perturbation and must be reported as a separate RLE instruction protocol.</p>' : ''}</aside>
      <div class="audit-snapshot-links"><span>Snapshot ${esc(audit.created_at.slice(0, 10))} · ${esc(audit.version)}</span>${link(sourceRoot + 'sources.json', 'Source hashes')}${link(sourceRoot + panel + '.json', 'Audit JSON')}</div></section>
      <section class="panel audit-controls" aria-label="Instruction audit filters"><div class="audit-summary" id="audit-summary"><strong>${rows.length} ${panel === 'plus' ? 'task/configuration pairs' : 'tasks'}</strong><span>${changed} changed</span><span>${rows.length - changed} unchanged</span><span>${countBindings} scene-bound goals</span></div>
      <div class="filters"><label class="search-field"><span aria-hidden="true">⌕</span><input id="audit-search" type="search" aria-label="Search instructions" placeholder="Search task, ID or either instruction"></label>
      <select id="audit-family" aria-label="Configuration"><option value="">All configurations</option>${audit.families.map(value => `<option value="${esc(value.id)}">${esc(value.name)}</option>`).join('')}</select>
      <select id="audit-status" aria-label="Instruction changes"><option value="">All instructions</option><option value="changed">Changed only</option><option value="unchanged">Unchanged only</option></select>
      <select id="audit-binding" aria-label="Goal binding"><option value="">All goal types</option><option value="bound">Scene-bound goals</option><option value="fixed">Fixed goals</option></select></div>
      <div class="audit-toolbar"><div class="audit-legend"><span><mark class="instruction-added">Added / revised</mark> in RLE</span><span><del class="instruction-removed">Removed / replaced</del> from native</span></div><span id="audit-count" role="status" aria-live="polite"></span><button class="text-button" id="audit-reset">Reset filters</button></div></section><div id="audit-groups"></div>`;

    function card(task) {
      const source = audit.native_sources[task.definition];
      const bindingEntries = Object.entries(task.bindings);
      const nativeCount = task.pool_keys.reduce((sum, key) => sum + source.pools[key].length, 0);
      const recorded = episodes.get(task.published_task_key) || [];
      return `<article class="panel audit-task" data-audit-task="${esc(task.key)}" data-status="${task.status}" data-audit-index="${task.index}">
        <header class="audit-task-heading"><div><div class="audit-task-id"><code>${esc(task.key)}</code><span>${esc(task.id)}</span></div><h3>${esc(task.title)}</h3></div><span class="audit-status ${task.status}">${task.status === 'changed' ? 'Changed' : 'Unchanged'}</span></header>
        <div class="audit-pair"><section class="audit-column measured"><h4>Current RLE instruction</h4><p class="audit-column-note">${bindingEntries.length ? 'Scene-bound goal template' : 'Fixed goal'} · ${esc(audit.family)}</p><div class="audit-text" data-audit-measured>${task.difference.revised}</div></section>
        <section class="audit-column native"><h4>Native task description</h4><p class="audit-column-note">Upstream full_description · ${esc(source.revision.slice(0, 8))}</p><div class="audit-text" data-audit-native>${task.difference.original}</div></section></div>
        ${task.notes.length ? `<ul class="audit-change-notes">${task.notes.map(note => `<li>${esc(note)}</li>`).join('')}</ul>` : ''}
        ${bindingEntries.length ? `<div class="audit-bindings"><strong>Scene bindings</strong>${bindingEntries.map(([name, path]) => `<span><code>{${esc(name)}}</code> ← <code>${esc(path)}</code></span>`).join('')}</div>` : ''}
        <details class="audit-language" data-native-pool="${task.index}"><summary>Native runtime language · ${task.pool_keys.map(key => esc(poolNames[key])).join(' / ')} · ${nativeCount} ${task.configuration === 'demo_language_plus' ? 'sentences' : 'templates'}</summary><div class="audit-language-content"></div></details>
        <details class="audit-language" data-recorded-language="${task.index}"><summary>Recorded episode instructions · ${recorded.length} episodes</summary><div class="audit-language-content"></div></details>
        <footer class="audit-task-footer"><span>${esc(task.configuration)}</span>${link(task.source_url, 'Native source')}${link(task.verifier_url, 'Native verifier')}${link(sourceRoot + task.rle_snapshot, 'RLE catalog snapshot')}${link(sourceRoot + task.rle_manifest, 'RLE manifest')}</footer></article>`;
    }

    function renderRows() {
      const query = filters.query.trim().toLowerCase();
      const shown = rows.filter(task => (!filters.family || task.family === filters.family) && (!filters.status || task.status === filters.status) &&
        (!filters.binding || Boolean(Object.keys(task.bindings).length) === (filters.binding === 'bound')) &&
        (!query || [task.key, task.id, task.title, task.native_task, task.configuration, task.native, task.measured].some(value => value.toLowerCase().includes(query))));
      main.querySelector('#audit-count').textContent = `${shown.length} / ${rows.length} tasks shown`;
      main.querySelector('#audit-groups').innerHTML = shown.length ? audit.families.map(family => {
        const matches = shown.filter(task => task.family === family.id);
        return matches.length ? `<section class="audit-family"><div class="audit-family-heading"><h2>${esc(family.name)}</h2><span>${matches.length} tasks</span></div>${matches.map(card).join('')}</section>` : '';
      }).join('') : '<div class="empty-state"><h2>No matching tasks</h2><p>Try another search or reset the filters.</p></div>';
      main.querySelectorAll('[data-native-pool]').forEach(details => details.addEventListener('toggle', () => {
        if (details.open && !details.dataset.loaded) {nativePool(details, rows[Number(details.dataset.nativePool)]); details.dataset.loaded = 'true';}
      }));
      main.querySelectorAll('[data-recorded-language]').forEach(details => details.addEventListener('toggle', () => {
        if (details.open && !details.dataset.loaded) {recordedLanguage(details, rows[Number(details.dataset.recordedLanguage)]); details.dataset.loaded = 'true';}
      }));
    }

    function nativePool(details, task) {
      const source = audit.native_sources[task.definition], container = details.querySelector('.audit-language-content');
      const values = task.pool_keys.flatMap(key => source.pools[key].map((text, index) => ({text, key, index})));
      container.innerHTML = `<p class="audit-column-note">${task.configuration === 'demo_language_plus' ? 'Complete native sentences from the R1/R2/R3 pool; retained verbatim.' : 'Upstream runtime templates. Native placeholders use the upstream schema and are filled from the episode.'}</p>
        ${source.schema ? `<p class="audit-native-schema">${esc(source.schema)}</p>` : ''}<label>Native sample<select aria-label="Native language sample">${values.map((value, index) => `<option value="${index}">${esc(poolNames[value.key])} · ${value.index + 1}</option>`).join('')}</select></label><div class="audit-text" data-native-sample></div>`;
      const show = () => {container.querySelector('[data-native-sample]').textContent = values[Number(container.querySelector('select').value)].text;};
      container.querySelector('select').addEventListener('change', show); show();
    }

    function recordedLanguage(details, task) {
      const values = episodes.get(task.published_task_key) || [], container = details.querySelector('.audit-language-content');
      if (!values.length) {container.textContent = 'No published episode for this task.'; return;}
      container.innerHTML = `<p class="audit-column-note">Exact recorded episode text. It is separate from the current catalog template above.</p><label>Published episode<select aria-label="Recorded instruction episode">${values.map((episode, index) => `<option value="${index}">${esc(variants.get(episode.variant) || episode.variant)} · seed ${esc(episode.seed)}</option>`).join('')}</select></label><div class="audit-text" data-recorded-instruction></div><div class="audit-recorded-source"></div>`;
      const show = () => {
        const episode = values[Number(container.querySelector('select').value)];
        container.querySelector('[data-recorded-instruction]').textContent = measuredInstructions.episodes[episode.id]?.text || episode.instruction || 'No recorded instruction text.';
        container.querySelector('.audit-recorded-source').innerHTML = `<span>Recorded policy: ${esc(episode.instruction_policy || 'unspecified')}</span><a href="#task/${esc(task.published_task_key)}/${esc(episode.variant)}/overview">Open episode evidence ↗</a>`;
      };
      container.querySelector('select').addEventListener('change', show); show();
    }

    main.querySelector('#audit-search').addEventListener('input', event => {filters.query = event.target.value; renderRows();});
    for (const key of ['family', 'status', 'binding']) main.querySelector('#audit-' + key).addEventListener('change', event => {filters[key] = event.target.value; renderRows();});
    main.querySelector('#audit-reset').addEventListener('click', () => {
      for (const key of Object.keys(filters)) filters[key] = '';
      main.querySelectorAll('.audit-controls input, .audit-controls select').forEach(element => {element.value = '';});
      renderRows();
    });
    renderRows();
  }
  return {render};
})();