fix: robust published article restore on container rebuild

- Add detailed logging to pullPublishedAssets for diagnostics
- Retry restore for anonymous visitors (dataset is public, no token needed)
- Add mutex to prevent concurrent pull attempts
- Add direct HTTP fetch fallback when Hub library fails
- Log SPACE_ID and HF_DATASET_ID at startup

Made-with: Cursor

backend/src/hf-storage.ts

@@ -149,40 +149,67 @@ export async function pullPublishedAssets(
   docName: string,
   localDir: string,
 ): Promise<boolean> {
-  if (!HF_DATASET_ID) return false;
+  if (!HF_DATASET_ID) {
+    console.log("[hf-storage] pullPublished skipped: no dataset ID");
+    return false;
+  }
 
-  // Token is optional: dataset is public, reads work without auth
-  const accessToken = getToken() || undefined;
+  const token = getToken();
+  const accessToken = token || undefined;
+  console.log(`[hf-storage] pullPublished for "${docName}" from dataset ${HF_DATASET_ID} (token: ${token ? "yes" : "none - public access"})`);
 
   const safeName = sanitizeName(docName);
   const base = `published/${safeName}`;
   const files = ["index.html", "article.pdf", "thumb.jpg", "meta.json"];
 
+  const { mkdirSync, writeFileSync } = await import("fs");
+  const { join } = await import("path");
+  const outDir = join(localDir, "published", safeName);
+
   let foundAny = false;
 
   for (const file of files) {
     try {
+      // Try Hub library first
       const res = await downloadFile({
         repo,
         path: `${base}/${file}`,
         accessToken,
       });
-      if (!res) continue;
+      if (!res) {
+        console.log(`[hf-storage] ${base}/${file}: not found via Hub lib`);
+        continue;
+      }
 
-      const { mkdirSync, writeFileSync } = await import("fs");
-      const { join } = await import("path");
-      const outDir = join(localDir, "published", safeName);
       mkdirSync(outDir, { recursive: true });
-
       const buf = Buffer.from(await res.arrayBuffer());
       writeFileSync(join(outDir, file), buf);
       console.log(`[hf-storage] pulled ${base}/${file} (${buf.length} bytes)`);
       foundAny = true;
-    } catch {
-      // File may not exist (e.g. PDF not generated); skip silently
+    } catch (err) {
+      console.warn(`[hf-storage] Hub lib failed for ${base}/${file}:`, (err as Error).message);
+
+      // Fallback: direct HTTP fetch (works for public datasets without auth)
+      try {
+        const url = `https://huggingface.co/datasets/${HF_DATASET_ID}/resolve/main/${base}/${file}`;
+        console.log(`[hf-storage] trying direct fetch: ${url}`);
+        const httpRes = await fetch(url);
+        if (!httpRes.ok) {
+          console.log(`[hf-storage] direct fetch ${file}: ${httpRes.status}`);
+          continue;
+        }
+        mkdirSync(outDir, { recursive: true });
+        const buf = Buffer.from(await httpRes.arrayBuffer());
+        writeFileSync(join(outDir, file), buf);
+        console.log(`[hf-storage] pulled ${base}/${file} via direct fetch (${buf.length} bytes)`);
+        foundAny = true;
+      } catch (fetchErr) {
+        console.warn(`[hf-storage] direct fetch also failed for ${file}:`, (fetchErr as Error).message);
+      }
     }
   }
 
+  console.log(`[hf-storage] pullPublished result: ${foundAny ? "restored" : "nothing found"}`);
   return foundAny;
 }
 

backend/src/server.ts

@@ -35,10 +35,12 @@ const DEFAULT_DOC_NAME = "default";
 
 /**
  * Lazy restore of published assets from HF dataset.
- * Called at startup (may fail without token) and again on first auth.
- * Only runs the actual pull once successfully.
+ * Called at startup and on every request until successful.
+ * Dataset is public so no token is required for reads.
  */
 let _publishedRestored = false;
+let _restoreInProgress: Promise<void> | null = null;
+
 async function ensurePublishedRestored(token?: string): Promise<void> {
   if (_publishedRestored) return;
   if (!getDatasetId()) return;
@@ -49,17 +51,29 @@ async function ensurePublishedRestored(token?: string): Promise<void> {
     return;
   }
 
+  // Prevent concurrent pulls (multiple requests arriving at the same time)
+  if (_restoreInProgress) {
+    await _restoreInProgress;
+    return;
+  }
+
   if (token) setUserToken(token);
 
-  try {
-    const found = await pullPublishedAssets(DEFAULT_DOC_NAME, DATA_DIR);
-    if (found) {
-      _publishedRestored = true;
-      console.log("[server] restored published article from HF dataset");
+  _restoreInProgress = (async () => {
+    try {
+      const found = await pullPublishedAssets(DEFAULT_DOC_NAME, DATA_DIR);
+      if (found) {
+        _publishedRestored = true;
+        console.log("[server] restored published article from HF dataset");
+      }
+    } catch (err) {
+      console.warn("[server] failed to restore published:", (err as Error).message);
+    } finally {
+      _restoreInProgress = null;
     }
-  } catch (err) {
-    console.warn("[server] failed to restore published:", (err as Error).message);
-  }
+  })();
+
+  await _restoreInProgress;
 }
 
 mkdirSync(DATA_DIR, { recursive: true });
@@ -384,11 +398,11 @@ if (existsSync(staticDir)) {
       return;
     }
 
-    // Try lazy-restore if we have a cookie token (visitor may be logged in)
+    // Try lazy-restore on every request until successful.
+    // Dataset is public, so no token is needed for reads.
+    // An OAuth token is passed when available for extra reliability.
     const visitorToken = extractToken(req.headers.cookie) ?? undefined;
-    if (visitorToken) {
-      await ensurePublishedRestored(visitorToken);
-    }
+    await ensurePublishedRestored(visitorToken);
 
     // In production with OAuth: serve the published article if it exists
     const publishedPath = getPublishedPath(DEFAULT_DOC_NAME);
@@ -408,8 +422,9 @@ if (existsSync(staticDir)) {
 const server = httpServer.listen(PORT, async () => {
   console.log(`[server] running on http://localhost:${PORT}`);
   console.log(`[server] collab websocket at ws://localhost:${PORT}/collab/:doc`);
+  console.log(`[server] SPACE_ID=${process.env.SPACE_ID || "(not set)"} HF_DATASET_ID=${getDatasetId() || "(empty)"} HF_TOKEN=${process.env.HF_TOKEN ? "set" : "not set"}`);
 
-  // Try eager restore (works if HF_TOKEN env var is set)
+  // Eager restore: dataset is public, works without token
   await ensurePublishedRestored();
 });