Fix production crash: bundle corpora.json in Git and fetch annotations from HF Dataset

annotation_data/corpora.json

@@ -0,0 +1,8 @@
+[
+    {
+        "id": "sample",
+        "name": "Sample PDFs",
+        "extractions_dir": "sample_extractions",
+        "links_file": "sample_data/sample_pdf_links.json"
+    }
+]

utils/storage.js

@@ -197,43 +197,74 @@ export async function updateAnnotation(timestamp, docIndex, pageNumber, updates,
 }
 
 /**
- * Retrieves all human annotations from local files.
+ * Retrieves all human annotations.
  */
 export async function getAnnotations(docIndex = null, corpusId = null) {
-    const { getCorpora } = await import('./config.js');
+    const { getCorpora, HF_DATASET_BASE_URL, getLinksRepoPath, getDocRepoPath } = await import('./config.js');
     const corporaList = corpusId ? [getCorpus(corpusId)] : getCorpora();
     const results = [];
 
     for (const corpus of corporaList) {
-        const extractionsDir = path.join(process.cwd(), 'annotation_data', corpus.extractions_dir);
-        if (!fs.existsSync(extractionsDir)) continue;
-
-        const dirs = fs.readdirSync(extractionsDir).filter(d => d.startsWith('doc_'));
-
-        for (const dir of dirs) {
-            const idx = parseInt(dir.replace('doc_', ''), 10);
-            if (docIndex !== null && idx !== docIndex) continue;
+        let links = [];
+        if (isHFSpace()) {
+            try {
+                const linksPath = getLinksRepoPath(corpus);
+                const linksUrl = `${HF_DATASET_BASE_URL}/raw/main/${linksPath}`;
+                const res = await fetch(linksUrl, {
+                    headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` },
+                    cache: 'no-store'
+                });
+                if (res.ok) {
+                    links = await res.json();
+                }
+            } catch (e) {
+                console.warn(`Error fetching links file for ${corpus.id} in getAnnotations:`, e.message);
+            }
+        } else {
+            const filePath = path.join(process.cwd(), 'annotation_data', corpus.links_file);
+            if (fs.existsSync(filePath)) {
+                links = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+            }
+        }
 
-            const filePath = path.join(extractionsDir, dir, 'raw', `${dir}_direct_judged.jsonl`);
-            if (!fs.existsSync(filePath)) continue;
+        const targetLinks = links.filter(l => l.status === 'success' && (docIndex === null || l.index === docIndex));
 
+        for (const link of targetLinks) {
+            let pagesData = null;
             try {
-                const pagesData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
-                for (const page of pagesData) {
-                    const pageNum = page.document?.pages?.[0];
-                    for (const ds of (page.datasets || [])) {
-                        if (ds.annotator) {
-                            results.push({
-                                ...ds,
-                                corpus: corpus.id,
-                                document_index: idx,
-                                page_number: pageNum,
-                            });
+                if (isHFSpace()) {
+                    const docRepoPath = getDocRepoPath(corpus, link.index);
+                    const docUrl = `${HF_DATASET_BASE_URL}/raw/main/${docRepoPath}`;
+                    const res = await fetch(docUrl, {
+                        headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` }
+                    });
+                    if (res.ok) {
+                        pagesData = await res.json();
+                    }
+                } else {
+                    const filePath = getDocLocalPath(corpus, link.index);
+                    if (fs.existsSync(filePath)) {
+                        pagesData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+                    }
+                }
+
+                if (pagesData) {
+                    for (const page of pagesData) {
+                        const pageNum = page.document?.pages?.[0];
+                        for (const ds of (page.datasets || [])) {
+                            if (ds.annotator) {
+                                results.push({
+                                    ...ds,
+                                    corpus: corpus.id,
+                                    document_index: link.index,
+                                    page_number: pageNum,
+                                });
+                            }
                         }
                     }
                 }
             } catch (e) {
-                console.error(`Error reading ${filePath}:`, e);
+                console.error(`Error loading annotations for doc_${link.index}:`, e);
             }
         }
     }