Upload folder using huggingface_hub

openenv/berth_openenv/environment.py

@@ -24,7 +24,7 @@ from typing import Annotated, Any
 from fastmcp import FastMCP
 from openenv.core.env_server.mcp_environment import MCPEnvironment
 from openenv.core.env_server.mcp_types import CallToolAction
-from openenv.core.env_server.types import Action, Observation, State
+from openenv.core.env_server.types import Action, EnvironmentMetadata, Observation, State
 from pydantic import BaseModel, Field
 
 from berth_core import PROMPT_VERSION, Task, TaskPack, evaluate, load_pack, parse_plan, plan_to_list, rules, situation
@@ -121,6 +121,18 @@ class BerthPlanningEnvironment(MCPEnvironment):
 
     SUPPORTS_CONCURRENT_SESSIONS = True
 
+    def get_metadata(self) -> EnvironmentMetadata:
+        """What OpenEnv's /metadata reports (the web UI shows it too)."""
+        return EnvironmentMetadata(
+            name="PortSimEnv v1",
+            description="Re-plan a broken week of container-ship dockings at a Port of Barcelona quay, built from the "
+                        "port's real 2024 calls. Tools: get_situation, check_plan (10 per episode), submit_plan "
+                        "(graded once, deterministically, against the CP-SAT optimum).",
+            version="1.0.0",
+            author="FineEnvs",
+            documentation_url="https://github.com/adithya-s-k/FineEnvs/tree/main/07-simulation-environments/portsim-v1",
+        )
+
     def __init__(self, pack: TaskPack | None = None):
         self.pack = pack or load_pack()
         self._episode: Episode | None = None

openenv/berth_openenv/gradio_ui.py

@@ -21,8 +21,8 @@ DEFAULT_SPLIT = "eval"
 DEFAULT_TASK = "dock-24B-w07x1-busy-0"
 
 
-def _iframe(task_id: str, split: str) -> str:
-    src = f"/viewer/?embed=1#/play?split={quote(split)}&task={quote(task_id)}&start=1"
+def _iframe(task_id: str, split: str, index: int) -> str:
+    src = f"/viewer/?embed=1#/play?split={quote(split)}&index={index}&task={quote(task_id)}&start=1"
     return (
         # OpenEnv's page caps the container width; the editor wants the full window.
         "<style>.gradio-container .main.fillable{max-width:none!important;padding-left:12px!important;"
@@ -44,7 +44,7 @@ def build_ui(pack: TaskPack) -> gr.Blocks:
     def _frame(index, split):
         split = split if split in ids else split0
         i = min(max(int(index or 0), 0), len(ids[split]) - 1)
-        return _iframe(ids[split][i], split)
+        return _iframe(ids[split][i], split, i)
 
     with gr.Blocks(title="PortSimEnv v1") as ui:
         with gr.Row():

openenv/berth_openenv/web/index.html

@@ -5,7 +5,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <title>PortSimEnv v1 · Port of Barcelona</title>
   <meta name="description" content="Re-planning a week of container-ship dockings at one quay of the Port of Barcelona after a disruption.">
-  <link rel="stylesheet" href="./style.css?v=eval-1">
+  <link rel="stylesheet" href="./style.css?v=eval-2">
   <script type="importmap">
   {
     "imports": {

openenv/berth_openenv/web/overview.js

@@ -55,6 +55,22 @@ function board(episodes) {
   })).sort((a, b) => b.mean - a.mean);
 }
 
+// Outcome of one rollout, for the bars and the grid.
+const OUTCOMES = [
+  { k: "optimal", label: "optimal plan (1.0)" },
+  { k: "valid", label: "valid, not optimal" },
+  { k: "broken", label: "broke a rule (≤ 0.2)" },
+  { k: "none", label: "no plan submitted (0)" },
+];
+const outcomeOf = (e) => (!e.submitted ? "none" : !e.feasible ? "broken" : (e.reward || 0) >= 0.999 ? "optimal" : "valid");
+
+function outcomeBar(r) {
+  const n = r.n || 1;
+  const c = { optimal: r.optimal, valid: r.valid - r.optimal, broken: r.submitted - r.valid, none: r.n - r.submitted };
+  return `<span class="ov-obar" title="${OUTCOMES.map((o) => `${c[o.k]} ${o.label}`).join(" · ")}">${OUTCOMES.filter((o) => c[o.k] > 0)
+    .map((o) => `<i class="oc-${o.k}" style="width:${((100 * c[o.k]) / n).toFixed(2)}%"></i>`).join("")}</span>`;
+}
+
 function resultHtml(e) {
   if (!e.submitted) return `<span class="muted" title="${escapeHtml(e.end_reason || "")}">not submitted</span>`;
   if (!e.feasible) return '<i class="dot bad"></i>rule broken';
@@ -113,11 +129,16 @@ export async function overviewPage({ app, setCrumbs, isCurrent, params, config =
   const evalSec = `<section>
           <div class="sec-head"><h2>Eval</h2><span class="muted small">${RUN} · 50 held-out tasks · one rollout per model and task · 12 turns, 32k output tokens per turn</span></div>
           <table class="tbl click" id="ov-board">
-            <thead><tr><th>Model</th><th class="num">Mean</th>${TIERS.map((t) => `<th class="num opt" title="${escapeHtml(TIER_TIPS[t])}">${cap(t)}</th>`).join("")}<th class="num" title="Episodes that ended with submit_plan">Submitted</th><th class="num" title="Plans that break no rule">Valid</th><th class="num" title="Plans that match the proven optimum (reward 1.0)">Optimal</th></tr></thead>
-            <tbody>${Array.from({ length: 6 }, () => `<tr class="skel">${"<td><i></i></td>".repeat(9)}</tr>`).join("")}</tbody>
+            <thead><tr><th>Model</th><th class="ov-oth" title="The 50 eval weeks by outcome">Outcomes</th><th class="num">Mean</th>${TIERS.map((t) => `<th class="num opt" title="${escapeHtml(TIER_TIPS[t])}">${cap(t)}</th>`).join("")}<th class="num" title="Episodes that ended with submit_plan">Submitted</th><th class="num" title="Plans that break no rule">Valid</th><th class="num" title="Plans that match the proven optimum (reward 1.0)">Optimal</th></tr></thead>
+            <tbody>${Array.from({ length: 6 }, () => `<tr class="skel">${"<td><i></i></td>".repeat(10)}</tr>`).join("")}</tbody>
           </table>
+          <div class="ov-legend">${OUTCOMES.map((o) => `<span><i class="oc-${o.k}"></i>${o.label}</span>`).join("")}</div>
           <p class="muted small ov-note" id="ov-board-note">Mean reward per model and tier. Click a model to list its rollouts; each one replays in 3D.</p>
         </section>
+        <section class="ov-grid-sec" hidden>
+          <div class="sec-head"><h2>Every rollout</h2><span class="muted small">one square per model and eval week, coloured by outcome (fainter = lower reward) · click a square to replay it in 3D</span></div>
+          <div class="ov-mx" id="ov-mx"></div>
+        </section>
         <section class="ov-eps" hidden>
           <div class="sec-head"><h2 id="ov-eps-h">Rollouts</h2><span class="muted small" id="ov-eps-note"></span></div>
           <table class="tbl click" id="ov-eps"><thead></thead><tbody></tbody></table>
@@ -198,7 +219,7 @@ print(step.reward)   # 0..1, graded once</code></pre>
 
   const tbody = app.querySelector("#ov-board tbody");
   const noRollouts = () => {
-    tbody.innerHTML = `<tr><td colspan="9" class="muted">No eval rollouts on this server.${explorer ? "" : ` They are in the <a href="${EVAL_VIEWER}" target="_blank" rel="noopener">eval Space ↗</a>.`}</td></tr>`;
+    tbody.innerHTML = `<tr><td colspan="10" class="muted">No eval rollouts on this server.${explorer ? "" : ` They are in the <a href="${EVAL_VIEWER}" target="_blank" rel="noopener">eval Space ↗</a>.`}</td></tr>`;
     app.querySelector("#ov-board-note").hidden = true;
     if (explorer) return;
     for (const a of app.querySelectorAll('a[href^="#/run/"]')) { // the showcase rollout: watch it in the eval Space
@@ -218,7 +239,7 @@ print(step.reward)   # 0..1, graded once</code></pre>
   if (!isCurrent()) return;
   if (!rows.length) return noRollouts();
   tbody.innerHTML = rows.map((r) => `<tr data-row="${escapeHtml(r.model)}" title="List ${escapeHtml(r.name)}'s rollouts">
-      <td>${escapeHtml(r.name)}</td><td class="num"><b>${fmtNum(r.mean, 3)}</b></td>
+      <td>${escapeHtml(r.name)}</td><td class="ov-otd">${outcomeBar(r)}</td><td class="num"><b>${fmtNum(r.mean, 3)}</b></td>
       ${TIERS.map((t) => `<td class="num opt">${r.tiers[t] == null ? "–" : fmtNum(r.tiers[t], 2)}</td>`).join("")}
       <td class="num">${r.submitted}/${r.n}</td><td class="num">${r.valid}/${r.n}</td><td class="num">${r.optimal}/${r.n}</td></tr>`).join("");
 
@@ -226,6 +247,24 @@ print(step.reward)   # 0..1, graded once</code></pre>
   for (const e of rows[0].eps) perTier.set(e.difficulty, (perTier.get(e.difficulty) || 0) + 1);
   app.querySelector("#ov-board-note").textContent = `Mean reward per model and tier (${TIERS.filter((t) => perTier.has(t)).map((t) => `${t} ${perTier.get(t)}`).join(", ")} tasks). Click a model to list its rollouts; each one replays in 3D.`;
 
+  // Every rollout at a glance: models down, eval weeks across (grouped by tier), each square a link to its 3D replay.
+  {
+    const tasks = [...new Map(rows.flatMap((r) => r.eps).map((e) => [e.task_id, e])).values()]
+      .sort((a, b) => TIERS.indexOf(a.difficulty) - TIERS.indexOf(b.difficulty) || a.task_id.localeCompare(b.task_id));
+    const byKey = new Map(rows.flatMap((r) => r.eps.map((e) => [`${r.model}|${e.task_id}`, e])));
+    const groups = TIERS.map((t) => ({ t, tasks: tasks.filter((e) => e.difficulty === t) })).filter((g) => g.tasks.length);
+    const cell = (r, task) => {
+      const e = byKey.get(`${r.model}|${task.task_id}`);
+      if (!e) return '<span class="mx-c mx-missing"></span>';
+      const oc = outcomeOf(e);
+      const op = oc === "valid" ? (0.35 + 0.65 * Math.max(0, ((e.reward || 0) - 0.2) / 0.8)).toFixed(2) : "1";
+      return `<a class="mx-c oc-${oc}" style="opacity:${op}" href="${runHref(r.model, task.task_id)}" title="${escapeHtml(r.name)} · ${escapeHtml(task.task_id)} · ${task.ships} ships · ${oc === "none" ? "no plan" : `reward ${fmtNum(e.reward, 2)}`}"></a>`;
+    };
+    app.querySelector("#ov-mx").innerHTML = `<div class="mx-row mx-head"><span class="mx-name"></span>${groups.map((g) => `<span class="mx-g" style="--n:${g.tasks.length}" title="${escapeHtml(TIER_TIPS[g.t])}">${cap(g.t)} · ${g.tasks.length}</span>`).join("")}</div>` +
+      rows.map((r) => `<div class="mx-row"><span class="mx-name">${escapeHtml(r.name)}</span>${groups.map((g) => `<span class="mx-g" style="--n:${g.tasks.length}">${g.tasks.map((t) => cell(r, t)).join("")}</span>`).join("")}<span class="mx-mean">${fmtNum(r.mean, 2)}</span></div>`).join("");
+    app.querySelector(".ov-grid-sec").hidden = false;
+  }
+
   // The selected model's rollouts, under the board (#/?model=<id> keeps the choice in shared links).
   const epsSec = app.querySelector(".ov-eps");
   let epsTable = null;

openenv/berth_openenv/web/play.js

@@ -1297,7 +1297,22 @@ export async function playPage({ app, params, isCurrent, setTeardown, sceneMod }
 
   onConnStatus(session.status);
   bindShell();
-  // #/play?task=<id>&start=1 starts that task's episode straight away (used by embeds)
+  // #/play?split=<s>&index=<i>&start=1 starts that task's episode straight away, without the picker (OpenEnv's /web
+  // tab and embeds use it); the one task comes from the Task API, so a 1,050-task split is never listed first.
+  const autoIndex = params.get("index");
+  if (params.get("start") === "1" && st.split && autoIndex !== null && autoIndex !== "") {
+    $("#pa-main").innerHTML = '<div class="pa-empty muted">Starting an episode…</div>';
+    taskApi("task", { split: st.split, index: Number(autoIndex) })
+      .then((d) => {
+        const t = d.task || d;
+        if (!isCurrent() || st.phase === "episode") return;
+        st.selected = t.task_id;
+        return start({ task_id: t.task_id }, t);
+      })
+      .catch(() => isCurrent() && openPicker());
+    return;
+  }
+  // #/play?task=<id>&start=1 (older links): pick it from the split list, then start
   openPicker().then(() => {
     if (params.get("start") !== "1" || !isCurrent() || st.phase === "episode") return;
     const t = pickerTasks.find((x) => x.task_id === st.selected);

openenv/berth_openenv/web/style.css

@@ -647,3 +647,25 @@ a.ext { font-size: 12px; white-space: nowrap; }
 #ov-eps td { white-space: nowrap; }
 .board-sec { overflow-x: auto; }
 @media (max-width: 700px) { html.explorer .crumbs { display: none; } }
+
+/* eval visuals: outcome bars and the rollout grid */
+.ov-oth { width: 30%; }
+.ov-otd { padding-top: 6px !important; }
+.ov-obar { display: flex; height: 12px; border-radius: 2px; overflow: hidden; background: var(--line2); min-width: 110px; }
+.ov-obar i { display: block; height: 100%; }
+.oc-optimal { background: var(--ok); }
+.oc-valid { background: color-mix(in srgb, var(--ok) 45%, var(--bg)); }
+.oc-broken { background: var(--red); }
+.oc-none { background: var(--faint); opacity: .45; }
+.ov-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; margin-top: 6px; font-size: 12px; color: var(--muted); }
+.ov-legend i { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin-right: 5px; vertical-align: -1px; }
+.ov-mx { overflow-x: auto; padding-bottom: 4px; }
+.mx-row { display: flex; align-items: center; gap: 10px; min-width: 760px; margin-bottom: 4px; }
+.mx-name { flex: 0 0 120px; font-size: 12px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+.mx-mean { flex: 0 0 34px; text-align: right; font-size: 12px; font-variant-numeric: tabular-nums; font-weight: 600; }
+.mx-g { flex: var(--n) 1 0; display: grid; grid-template-columns: repeat(var(--n), 1fr); gap: 2px; min-width: 0; }
+.mx-head .mx-g { display: block; font-size: 11px; color: var(--muted); border-bottom: 1px solid var(--line); padding-bottom: 2px; white-space: nowrap; overflow: hidden; }
+.mx-c { display: block; aspect-ratio: 1; border-radius: 2px; min-width: 8px; }
+.mx-c:hover { outline: 2px solid var(--fg); outline-offset: 1px; }
+.mx-missing { background: transparent; border: 1px dashed var(--line); }
+@media (max-width: 700px) { .ov-oth, .ov-otd { display: none; } }