Supervisor: Pi Web starts before projects are unpacked; session index for the launcher

README.md

@@ -22,7 +22,7 @@ What is inside:
 - Node 22, Python 3, uv, git, git-lfs, build tools, ripgrep, fd, jq, zstd
 - the `hf` CLI (`huggingface_hub` 2.1.1) and the Hugging Face agent skills in `~/.agents/skills`
 - `sandbox/hfpi.mjs`: the supervisor that restores the user's projects and sessions from their bucket, runs Pi Web,
-  saves state while it changes, and stops the machine when idle
+  saves state while it changes (with a session index the Space's launcher reads), and stops the machine when idle
 - `sandbox/git-credential-huggingface`: git authenticates to huggingface.co with the token the user lent at sign-in
 - `sandbox/AGENTS.md`: what the agent is told about this environment
 

sandbox/hfpi.mjs

@@ -14,6 +14,7 @@
 //   hfpi changes --name N          files changed since the import, for a Hub PR
 //   hfpi patch --name N            the same changes as a git patch
 //   hfpi status                    JSON for the gateway
+//   hfpi sessions                  the session index (also saved to the bucket with the agent)
 //   hfpi heartbeat                 user activity seen by the gateway
 //   hfpi stop                      final snapshot, then stop Pi Web and the supervisor
 
@@ -30,6 +31,7 @@ const RUN_DIR = process.env.HFPI_RUN_DIR || "/tmp/hfpi";
 const PORT = Number(process.env.HFPI_PIWEB_PORT || 30141);
 const PROJECTS_DIR = path.join(PERSIST, "projects");
 const AGENT_ARCHIVE = path.join(PERSIST, "agent", "agent.tar.zst");
+const SESSION_INDEX = path.join(PERSIST, "agent", "index.json"); // read by the Space while the machine is off
 const MACHINE_DIR = path.join(PERSIST, "machine");
 const IDLE_MINUTES = Number(process.env.HFPI_IDLE_MINUTES ?? 30);
 const MAX_LIFETIME_MINUTES = Number(process.env.HFPI_MAX_LIFETIME_MINUTES ?? 23 * 60 + 40);
@@ -256,10 +258,104 @@ function snapshotAgent({ force = false } = {}) {
     touch(pending);
     const bytes = writeArchive(AGENT_ARCHIVE, ["-C", AGENT_DIR, ...AGENT_EXCLUDES.map((e) => `--exclude=${e}`), "."]);
     fs.renameSync(pending, marker);
+    try {
+      writeSessionIndex();
+    } catch (e) {
+      log("session index failed:", e.message);
+    }
     return { bytes, at: new Date().toISOString() };
   });
 }
 
+// ---------------------------------------------------------------- session index
+
+// One line per session for the Space's launcher: it lists sessions instantly, before the
+// machine (and Pi Web) is up. Files are only re-read when they changed.
+const indexCache = new Map();
+let lastIndex = "";
+
+function messageText(message) {
+  const content = message?.content;
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) return content.filter((c) => c?.type === "text").map((c) => c.text).join(" ");
+  return "";
+}
+
+function sessionEntry(file) {
+  const st = fs.statSync(file);
+  const cached = indexCache.get(file);
+  if (cached && cached.mtimeMs === st.mtimeMs && cached.size === st.size) return cached.entry;
+  let header = null;
+  let name = null;
+  let firstUser = null;
+  let messages = 0;
+  for (const line of fs.readFileSync(file, "utf8").split("\n")) {
+    if (!line) continue;
+    if (!header) {
+      try {
+        header = JSON.parse(line);
+      } catch {
+        break;
+      }
+      continue;
+    }
+    if (line.includes('"type":"session_info"')) {
+      try {
+        name = JSON.parse(line).name || name;
+      } catch {}
+    } else if (line.startsWith('{"type":"message"')) {
+      messages++;
+      if (!firstUser && line.includes('"role":"user"')) {
+        try {
+          const entry = JSON.parse(line);
+          if (entry.message?.role === "user") firstUser = messageText(entry.message);
+        } catch {}
+      }
+    }
+  }
+  if (!header || header.type !== "session") return null;
+  const cwd = header.cwd || "";
+  const rel = path.relative(WORKSPACE, cwd);
+  const project = cwd.includes(`${path.sep}pi-cwd${path.sep}`) ? "pi-cwd" : !rel || rel.startsWith("..") ? path.basename(cwd) : rel.split(path.sep)[0];
+  const entry = {
+    id: header.id,
+    cwd,
+    project,
+    title: (name || firstUser || "").replace(/\s+/g, " ").trim().slice(0, 140) || "New session",
+    created: header.timestamp,
+    modified: st.mtimeMs,
+    messages,
+  };
+  indexCache.set(file, { mtimeMs: st.mtimeMs, size: st.size, entry });
+  return entry;
+}
+
+function sessionIndex(limit = 200) {
+  const root = path.join(AGENT_DIR, "sessions");
+  if (!exists(root)) return [];
+  const sessions = [];
+  for (const dir of fs.readdirSync(root, { withFileTypes: true })) {
+    if (!dir.isDirectory()) continue;
+    for (const file of fs.readdirSync(path.join(root, dir.name))) {
+      if (!file.endsWith(".jsonl")) continue;
+      try {
+        const entry = sessionEntry(path.join(root, dir.name, file));
+        if (entry) sessions.push(entry);
+      } catch {}
+    }
+  }
+  sessions.sort((a, b) => b.modified - a.modified);
+  return sessions.slice(0, limit);
+}
+
+function writeSessionIndex() {
+  const sessions = sessionIndex();
+  const text = JSON.stringify(sessions);
+  if (text === lastIndex && exists(SESSION_INDEX)) return;
+  writeJson(SESSION_INDEX, { updated_at: new Date().toISOString(), sessions });
+  lastIndex = text;
+}
+
 function snapshotAll({ force = false } = {}) {
   const results = { agent: null, projects: [] };
   try {
@@ -300,7 +396,7 @@ function restoreProjects() {
     if (!NAME_RE.test(name)) continue;
     const archive = projectArchivePath(name);
     const dir = path.join(WORKSPACE, name);
-    if (!exists(archive) || exists(dir)) continue;
+    if (!exists(archive) || (exists(dir) && fs.readdirSync(dir).length > 0)) continue;
     const started = Date.now();
     run("tar", ["--zstd", "-xf", archive, "-C", WORKSPACE]);
     touch(path.join(RUN_DIR, "markers", `project-${name}`));
@@ -414,10 +510,10 @@ async function cmdRun() {
   writeJson(path.join(RUN_DIR, "supervisor.json"), { pid: process.pid, started_at: new Date(bootStarted).toISOString(), phase: "restoring" });
   touch(path.join(RUN_DIR, "heartbeat"));
 
+  // Sessions and settings first, then Pi Web, which boots in its own process while the
+  // projects are unpacked here: Pi Web is ready seconds earlier.
   restoreAgent();
-  const restored = restoreProjects();
   configurePi();
-  log(`restore done in ${((Date.now() - bootStarted) / 1000).toFixed(1)}s (${restored.length} projects)`);
 
   const jobId = process.env.JOB_ID || "";
   const allowedHosts = [process.env.PI_WEB_ALLOWED_HOSTS, jobId && `${jobId}--${PORT}.hf.jobs`].filter(Boolean).join(",");
@@ -451,6 +547,8 @@ async function cmdRun() {
     });
   };
   startPiWeb();
+  const restored = restoreProjects();
+  log(`restore done in ${((Date.now() - bootStarted) / 1000).toFixed(1)}s (${restored.length} projects)`);
   writeJson(path.join(RUN_DIR, "supervisor.json"), { pid: process.pid, started_at: new Date(bootStarted).toISOString(), phase: "running", restored });
 
   const shutdown = async (reason) => {
@@ -689,6 +787,8 @@ async function main() {
       return cmdPatch(args);
     case "status":
       return print(await cmdStatus());
+    case "sessions":
+      return print({ sessions: sessionIndex() });
     case "heartbeat":
       touch(path.join(RUN_DIR, "heartbeat"));
       return print({ ok: true });
@@ -696,7 +796,7 @@ async function main() {
       touch(path.join(RUN_DIR, "stop-requested"));
       return print({ ok: true });
     default:
-      console.error("usage: hfpi run|snapshot|import|delete|changes|patch|status|heartbeat|stop");
+      console.error("usage: hfpi run|snapshot|import|delete|changes|patch|status|sessions|heartbeat|stop");
       process.exit(2);
   }
 }