Merge branch 'main' of https://huggingface.co/spaces/CGIAR/PERI-AI

app_web.py

@@ -1,5 +1,6 @@
 """Streamlit web UI for the RAG Research Chatbot."""
 import os
+import shutil
 import zipfile
 import streamlit as st
 
@@ -18,10 +19,11 @@ from huggingface_hub import snapshot_download
 
 snapshot_download(repo_id="CGIAR/peri-kb", 
                   repo_type="dataset", 
+                  allow_patterns="ug/*",
                   token=os.getenv('HF_TOKEN'), 
                   local_dir="./"
                   )
-
+shutil.copytree("./ug", "./", dirs_exist_ok=True)
 # ── Session initialisation ───────────────────────────────────────────────────
 
 def init_session():

src/ingest.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 import chromadb
 from chromadb.utils import embedding_functions
-from sentence_transformers import SentenceTransformer
+# from sentence_transformers import SentenceTransformer
 from tqdm import tqdm
 
 from src.readers import read_file, READERS

src/prompts.py

@@ -280,7 +280,217 @@ Output:
 USER'S QUESTION: {query}
 Return only the JSON object. Do not provide explanations, markdown, or additional text.
 """
-    
+
+VALUE_CHAIN_PROMPT_TEMPLATE = """\
+You are an expert policy and public expenditure analyst for a {domain} research chatbot.
+
+Your task is to review retrieved documents from a knowledge base and extract evidence related to government commitment to a specified agricultural value chain.
+
+IMPORTANT RULES
+
+1. Only use information explicitly found in the provided documents.
+2. Do not infer facts that are not supported by evidence.
+3. If evidence is not found, return "NOT_FOUND".
+4. Every extracted value must be accompanied by:
+   - supporting quotation
+   - source document name
+   - page/section reference when available
+5. Multiple documents may be provided.
+6. Consolidate information across all documents.
+7. Return output strictly as valid JSON.
+
+--------------------------------------------------------
+TARGET VALUE CHAIN
+--------------------------------------------------------
+
+Value Chain: {value_chain}
+
+--------------------------------------------------------
+EXTRACTION TASKS
+--------------------------------------------------------
+
+STEP 1: FINANCIAL COMMITMENT
+
+Search for budget allocations, expenditure allocations, projects, programs, or public investments associated with the value chain.
+
+Identify:
+
+A. Budget allocation during FY2020/2021 (or nearest baseline year)
+
+B. Budget allocation during FY2025/2026
+   (or most recent available year)
+
+Extract:
+
+- budget_start_year
+- budget_start_value
+- budget_end_year
+- budget_end_value
+- currency
+- responsible institution
+
+If multiple values are found, report all candidate values and explain differences.
+
+--------------------------------------------------------
+
+STEP 2: STRATEGIC COMMITMENT
+
+Across all supplied policy documents determine:
+
+1. Total number of documents examined
+
+2. Number of documents where the value chain is discussed
+
+3. Whether the value chain is discussed in:
+
+   a) Most recent National Development Plan
+
+   b) Most recent Agricultural Development Strategy,
+      Agricultural Sector Plan, Agricultural
+      Transformation Strategy, or equivalent
+
+   c) Standalone value chain strategy
+
+   d) Flagship value chain project
+
+For each item provide:
+
+- YES
+- NO
+- NOT_FOUND
+
+and supporting evidence.
+
+Definitions:
+
+National Development Plan:
+A country's primary medium-term national development policy document.
+
+Agricultural Strategy:
+The most recent agricultural sector strategy, investment plan, agricultural transformation plan, or equivalent.
+
+Standalone Value Chain Strategy:
+A policy, roadmap, strategy, investment plan, master plan, or action plan dedicated primarily to the target value chain.
+
+Flagship Project:
+A major government-supported, donor-supported, or co-financed project whose main focus is the target value chain.
+
+--------------------------------------------------------
+
+STEP 3: INSTITUTIONAL COMMITMENT
+
+Determine whether the value chain has:
+
+A. Dedicated government agency or parastatal
+
+B. Dedicated research institute or research center
+
+C. Dedicated ministry department,
+   directorate, authority, or unit
+
+Return:
+
+YES
+NO
+NOT_FOUND
+
+for each category.
+
+--------------------------------------------------------
+
+STEP 4: DOCUMENT COVERAGE
+
+Compute:
+
+documents_examined
+
+documents_discussing_value_chain
+
+coverage_weight
+
+Coverage Weight Formula:
+
+documents_discussing_value_chain /
+documents_examined
+
+--------------------------------------------------------
+
+OUTPUT FORMAT
+--------------------------------------------------------
+
+{
+  "value_chain": "",
+
+  "financial_commitment": {
+    "budget_start_year": "",
+    "budget_start_value": "",
+    "budget_end_year": "",
+    "budget_end_value": "",
+    "currency": "",
+    "responsible_institution": "",
+    "evidence": [
+      {
+        "quote": "",
+        "source_document": "",
+        "page_or_section": ""
+      }
+    ]
+  },
+
+  "strategic_commitment": {
+    "documents_examined": 0,
+    "documents_discussing_value_chain": 0,
+    "coverage_weight": 0,
+
+    "development_plan": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    },
+
+    "agricultural_strategy": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    },
+
+    "standalone_vc_strategy": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    },
+
+    "flagship_project": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    }
+  },
+
+  "institutional_commitment": {
+    "dedicated_agency": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    },
+
+    "dedicated_research_institute": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    },
+
+    "dedicated_ministerial_department": {
+      "value": "YES|NO|NOT_FOUND",
+      "evidence": []
+    }
+  }
+}
+
+"""
+
+def build_vc_prompt(value_chain:str,
+                    domain:str
+                   ):
+    return VALUE_CHAIN_PROMPT_TEMPLATE.format(
+        domain=domain,
+        value_chain=value_chain,
+    )
+
 def dict_to_string(d, indent=0):
     result = ""
     spacing = "  " * indent