File size: 12,479 Bytes
3bc016b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
<div class="harbor-task" style="width:100%;margin:14px 0;"></div>
<style>
  .harbor-task { position: relative; --ht-accent: rgb(78,165,183); }
  .harbor-task .ht-tasks { display:flex; flex-wrap:wrap; gap:6px; margin-bottom:10px; }
  .harbor-task .ht-chip { appearance:none; cursor:pointer; font:inherit; font-size:12px; line-height:1; padding:7px 12px;
    border-radius:999px; border:1px solid var(--border-color); background:transparent; color:var(--muted-color); }
  .harbor-task .ht-chip:hover { color:var(--text-color); }
  .harbor-task .ht-chip[aria-pressed="true"] { color:var(--text-color); border-color:var(--text-color); font-weight:600; }
  .harbor-task .ht-chip:focus-visible, .harbor-task .ht-file:focus-visible { outline:2px solid var(--text-color); outline-offset:2px; }
  .harbor-task .ht-q { font-size:13px; line-height:1.5; color:var(--text-color); margin:0 0 12px; }
  .harbor-task .ht-q span { color:var(--muted-color); }
  .harbor-task .ht-q code { font-size:12px; }
  .harbor-task .ht-body { display:grid; grid-template-columns: 230px 1fr; gap:12px; align-items:start; }
  .harbor-task .ht-tree { border:1px solid var(--border-color); border-radius:10px; padding:8px; }
  .harbor-task .ht-root { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; font-size:11px; color:var(--muted-color); padding:2px 6px 8px; word-break:break-all; }
  .harbor-task .ht-file { appearance:none; display:block; width:100%; text-align:left; cursor:pointer; font:inherit; background:none;
    border:0; border-radius:7px; padding:6px 8px; color:var(--muted-color); }
  .harbor-task .ht-file:hover { color:var(--text-color); }
  .harbor-task .ht-file[aria-selected="true"] { color:var(--text-color); background:color-mix(in srgb, var(--text-color) 7%, transparent); }
  .harbor-task .ht-file .n { display:block; font-family:ui-monospace,SFMono-Regular,Menlo,monospace; font-size:12px; }
  /* Folders are labels, not buttons; the files inside them are indented under a guide line. */
  .harbor-task .ht-dir { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; font-size:12px; color:var(--muted-color); padding:6px 8px 2px; }
  .harbor-task .ht-dir::before { content:'▾ '; opacity:.6; }
  .harbor-task .ht-in { margin-left:14px; padding-left:8px; border-left:1px solid var(--border-color); }
  .harbor-task .ht-file .r { display:block; font-size:11px; margin-top:1px; opacity:.85; }
  .harbor-task .ht-view { border:1px solid var(--border-color); border-radius:10px; overflow:hidden; min-width:0; }
  .harbor-task .ht-head { padding:9px 12px; border-bottom:1px solid var(--border-color); font-size:12.5px; line-height:1.5; color:var(--text-color); }
  .harbor-task .ht-head b { font-weight:600; }
  .harbor-task .ht-head code { font-size:11.5px; margin-right:4px; }
  .harbor-task .ht-head .p { float:right; font-family:ui-monospace,SFMono-Regular,Menlo,monospace; font-size:11px; color:var(--muted-color); margin-left:10px; }
  .harbor-task pre.ht-code { margin:0; padding:10px 0; max-height:360px; overflow:auto; font-size:11.5px; line-height:1.55;
    font-family:ui-monospace,SFMono-Regular,Menlo,monospace; background:color-mix(in srgb, var(--text-color) 3%, transparent); counter-reset:ln; }
  .harbor-task pre.ht-code .l { display:block; padding:0 12px 0 0; white-space:pre; }
  .harbor-task pre.ht-code .l::before { counter-increment:ln; content:counter(ln); display:inline-block; width:3.2em; padding-right:1em;
    text-align:right; color:var(--muted-color); opacity:.55; user-select:none; }
  .harbor-task .c { color:var(--muted-color); }
  .harbor-task .s { color:var(--ht-sec); font-weight:600; }
  .harbor-task .k { color:var(--ht-key); }
  .harbor-task .hl { background:color-mix(in srgb, var(--ht-key) 16%, transparent); border-radius:3px; }
  .harbor-task .ht-flow { display:flex; flex-wrap:wrap; align-items:center; gap:6px; margin-top:12px; font-size:12px; color:var(--text-color); }
  .harbor-task .ht-flow .t { font-size:11px; letter-spacing:.08em; text-transform:uppercase; color:var(--muted-color); width:100%; margin-bottom:2px; }
  .harbor-task .ht-flow .st { border:1px solid var(--border-color); border-radius:7px; padding:5px 9px; line-height:1.35; }
  .harbor-task .ht-flow .st code { font-size:11px; }
  .harbor-task .ht-flow .ar { color:var(--muted-color); }
  .harbor-task .nt { margin-top:10px; font-size:12px; line-height:1.5; color:var(--muted-color); }
  .harbor-task .nt code { font-size:11px; }
  /* prose files wrap, code keeps its lines */
  .harbor-task pre.ht-code.wrap .l { white-space:pre-wrap; padding-left:4.2em; text-indent:-4.2em; }
  @media (max-width: 640px) {
    .harbor-task .ht-body { grid-template-columns: 1fr; }
    .harbor-task .ht-tree { display:flex; overflow-x:auto; gap:4px; padding:6px; }
    .harbor-task .ht-root { display:none; }
    .harbor-task .ht-file { flex:none; width:auto; }
    .harbor-task .ht-file .r { display:none; }
    .harbor-task .ht-dir { display:none; }
    .harbor-task .ht-in { display:contents; }
  }
</style>
<script>
  (() => {
    const C = 'harbor-task';
    const boot = () => {
      const scriptEl = document.currentScript;
      let r = scriptEl ? scriptEl.previousElementSibling : null;
      while (r && !(r.classList && r.classList.contains(C))) r = r.previousElementSibling;
      if (!r) { const n = [...document.querySelectorAll('.'+C)].filter(e => e.dataset.mounted !== 'true'); r = n[n.length-1]; }
      if (!r || r.dataset.mounted === 'true') return;
      r.dataset.mounted = 'true';

      const P = (i) => {
        if (window.ColorPalettes && window.ColorPalettes.categorical) {
          try { return window.ColorPalettes.categorical(4)[i%4]; } catch(e) {}
        }
        return ['rgb(78,165,183)','rgb(146,176,234)','rgb(206,192,250)','rgb(232,137,171)'][i%4];
      };
      r.style.setProperty('--ht-sec', P(3));
      r.style.setProperty('--ht-key', P(0));
      const esc = (t) => String(t).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');

      // What each file is for. The contents come verbatim from the dataset on the Hub.
      const FILES = [
        ['task.toml',                  'the spec',   'The question, the gold answer and difficulty, the sandbox’s resources, the pre-agent hook, and the verifier’s settings.'],
        ['instruction.md',             'the prompt', 'Exactly what the agent is shown: where its files are, what is installed, and how to write the answer.'],
        ['environment/Dockerfile',     'the image',  'What the sandbox is built from. Every task in the suite shares one base image.'],
        ['environment/pull_bucket.py', 'the data',   'Runs from the healthcheck after the container starts and before the agent does, and downloads this task’s Kaggle files into /home/user/input.'],
        ['tests/test.sh',              'the verifier', 'Reads the agent’s answer. If there is none it writes 0.0 and exits cleanly, so a missing answer scores as wrong rather than as a crash.'],
        ['tests/grader.py',            'the reward', 'Exact match, then numeric tolerance, then order-insensitive lists, then symbolic comparison. It prints one float, and no LLM is involved.'],
      ];
      const LANG = { 'task.toml':'toml', 'instruction.md':'md', 'environment/Dockerfile':'sh', 'environment/pull_bucket.py':'py', 'tests/test.sh':'sh', 'tests/grader.py':'py' };

      // Light highlighting: comments, TOML sections and keys. The gold answer is marked
      // wherever it appears, so it is easy to see where the verifier gets it from.
      const paint = (text, lang, gold) => text.replace(/\n$/, '').split('\n').map(line => {
        let h = esc(line);
        const tr = line.trim();
        if ((lang === 'toml' || lang === 'sh' || lang === 'py') && tr.startsWith('#')) h = `<span class="c">${h}</span>`;
        else if (lang === 'toml' && /^\[.*\]$/.test(tr)) h = `<span class="s">${h}</span>`;
        else if (lang === 'toml') h = h.replace(/^(\s*)([A-Za-z_][\w.-]*)(\s*=)/, '$1<span class="k">$2</span>$3');
        if (gold && lang === 'toml') h = h.split(esc(`"${gold}"`)).join(`<span class="hl">${esc(`"${gold}"`)}</span>`);
        return `<span class="l">${h || ' '}</span>`;
      }).join('');

      // Files at the top of the task, then one block per folder with its files indented.
      const tree = () => {
        const btn = ([p, ro], i) => `<button type="button" role="tab" class="ht-file" data-f="${i}" aria-selected="${i===fi}" aria-label="${esc(p)}"><span class="n">${esc(p.split('/').pop())}</span><span class="r">${esc(ro)}</span></button>`;
        let h = '', dir = null;
        FILES.forEach((f, i) => {
          const d = f[0].includes('/') ? f[0].split('/')[0] : null;
          if (d !== dir) { if (dir) h += `</div>`; if (d) h += `<div class="ht-dir">${esc(d)}/</div><div class="ht-in">`; dir = d; }
          h += btn(f, i);
        });
        if (dir) h += `</div>`;
        return h;
      };

      let D = null, ti = 1, fi = 1;
      const ROTATE = [1, 4];        // instruction.md and tests/test.sh
      let rotation = 0;
      const stopRotating = () => { if (rotation) { clearInterval(rotation); rotation = 0; } };
      const draw = () => {
        if (!D) return;
        const T = D.tasks[ti];
        const [path, role, what] = FILES[fi];
        const body = T.files[path] ?? D.shared[path];
        const shared = !(path in T.files);
        r.innerHTML =
            `<div class="ht-tasks" role="group" aria-label="Choose a task">`
          + D.tasks.map((t,i) => `<button type="button" class="ht-chip" data-t="${i}" aria-pressed="${i===ti}">${t.tier[0].toUpperCase()+t.tier.slice(1)}, level ${t.level}</button>`).join('')
          + `</div>`
          + `<p class="ht-q">${esc(T.question)} <span>Gold answer:</span> <code>${esc(T.answer)}</code> <span>data from</span> <code>${esc(T.kaggle)}</code></p>`
          + `<div class="ht-body">`
          +   `<div class="ht-tree" role="tablist" aria-label="Files in this task">`
          +     `<div class="ht-root">tasks/${esc(T.id)}/</div>`
          +     tree()
          +   `</div>`
          +   `<div class="ht-view" role="tabpanel">`
          +     `<div class="ht-head"><span class="p">${shared ? 'same in every task' : 'this task only'}</span><code>${esc(path)}</code> <b>${esc(role[0].toUpperCase()+role.slice(1))}.</b> ${esc(what)}</div>`
          +     `<pre class="ht-code${LANG[path] === 'md' ? ' wrap' : ''}" tabindex="0" aria-label="${esc(path)}">${paint(body, LANG[path], T.answer)}</pre>`
          +   `</div>`
          + `</div>`
          + `<div class="ht-flow"><div class="t">How the reward gets out</div>`
          +   [`agent writes <code>/workdir/answer.txt</code>`, `<code>test.sh</code> reads it`, `<code>grader.py</code> prints one float`, `lands in <code>/logs/verifier/reward.txt</code>`, `Harbor returns it with the trial`]
                .map(x => `<span class="st">${x}</span>`).join('<span class="ar">→</span>')
          + `</div>`
          + `<div class="nt">Real files from <a href="https://huggingface.co/datasets/${esc(D.source)}" target="_blank" rel="noopener">${esc(D.source)}</a>, shown unedited. The gold answer is highlighted where <code>task.toml</code> hands it to the verifier.</div>`;
        r.querySelectorAll('.ht-chip').forEach(b => b.addEventListener('click', () => { stopRotating(); ti = +b.dataset.t; draw(); }));
        r.querySelectorAll('.ht-file').forEach(b => b.addEventListener('click', () => { stopRotating(); fi = +b.dataset.f; draw(); r.querySelector(`.ht-file[data-f="${fi}"]`)?.focus(); }));
      };

      const grab = async (f) => {
        for (const u of ['/data/'+f, 'assets/data/'+f, '../assets/data/'+f]) {
          try { const x = await fetch(u, {cache:'no-cache'}); if (x.ok) return await x.json(); } catch(e) {}
        }
        throw new Error('could not load '+f);
      };
      grab('harbor-tasks.json').then(d => {
        D = d; draw();
        if (!matchMedia('(prefers-reduced-motion: reduce)').matches) {
          let k = 0;
          rotation = setInterval(() => {
            k += 1;
            if (k >= ROTATE.length) { stopRotating(); return; }
            fi = ROTATE[k]; draw();
          }, 26000);
        }
      })
        .catch(e => { r.innerHTML = `<pre style="color:#c33;font-size:12px">${esc(e.message)}</pre>`; });
    };
    if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', boot, { once: true }); else boot();
  })();
</script>