update value chain prompts + output verification

app_web.py

@@ -16,7 +16,7 @@ from src.stata import explain_
 from src.peri import APPROVED_OPTIONS
 from src.peri.value_chains import categorize_vc_production, analyze_committment_to_vc , analyze_vc_committment
 from src.peri.investments import analyze_investments
-from src.prompts import dict_to_string
+from src.prompts import dict_to_string, VC_RAG
 # from ui import render_db
 from huggingface_hub import snapshot_download
 
@@ -128,23 +128,40 @@ def render_landing_page():
 
                 c2vc = {}
                 for vc in pol_geo_ref["value_chain"].unique():
-                    status.update(label=f'Analyzing committment to {vc} production...')
                     prompt = f"Look up and retrieve all relevant information on {vc} production in {country} from the knowledgebase."     
-                    qu_result = understand_query(prompt, cfg)
-
-                    search_query = qu_result.get("search_query", prompt)
-                    display_query = qu_result.get("display_query", prompt)
-                    sql_query = qu_result.get("sql_query")
-                    print(sql_query)
-
-                    retrieval_result = retrieve(search_query, cfg, route="both", sql_query=sql_query)
-                    st.session_state.last_retrieval = retrieval_result
-
-                    output = verify_and_respond(
-                        display_query, retrieval_result, cfg, vc, prompt,
-                    )
-                    print(output["response"])
-                    c2vc.update(_parse_vc_analyzer_result(output["response"]))
+                    status.update(label=f'Analyzing committment to {vc} production...')
+                    # for it in iterations:
+                    retrieval_results = {}
+                    for var, txt in VC_RAG.items():
+                        # status.update(label=f'Analyzing {var.replace("_", " ")} to {vc} production...')
+                        rag_prompt = txt.format(value_chain=vc)
+                        qu_result = understand_query(rag_prompt, cfg)
+
+                        search_query = qu_result.get("search_query", rag_prompt)
+                        display_query = qu_result.get("display_query", rag_prompt)
+                        sql_query = qu_result.get("sql_query")
+                        print(sql_query)
+
+                        if var=="strategic_commitment":
+                            retrieval_result = retrieve(rag_prompt,
+                                                            cfg,
+                                                            route="both",
+                                                            sql_query=sql_query,
+                                                            db_filter=cfg["policy_docs"])
+                            retrieval_result["policy_docs"] = set([os.path.basename(r["metadata"]["source"])
+                            for r in retrieval_result["db_results"]])
+
+                        else:
+                            retrieval_result = retrieve(rag_prompt, cfg, route="both", sql_query=sql_query)
+
+                        retrieval_result["var"] = var
+                        output = verify_and_respond(
+                            prompt, retrieval_result, cfg, vc, prompt
+                        )
+                        retrieval_results[var] = _parse_vc_analyzer_result(output["response"])
+                    c2vc[vc] = retrieval_results
+                    #     retrieval_results.append(output)
+                    # c2vc.update(_parse_vc_analyzer_result(output["response"]))
 
                 status.update(
                     label=f"Completed metadata preparation for {pol_geo_scores.shape[0]} value chains and {investments.shape[0]} investment areas in {country}.",

src/prompts.py

@@ -84,6 +84,464 @@ CONTEXT (use ONLY this to answer)
 {context}
 =====================================================================
 """
+FC_VERIFICATION_PROMPT_TEMPLATE = """\
+--- RESPONSE TO VERIFY ---
+{response}
+--- END RESPONSE ---
+
+--- CONTEXT ---
+{context}
+--- END CONTEXT ---
+
+--- PHRASE-LEVEL FLAGS ---
+{phrase_flags}
+--- END PHRASE FLAGS ---
+
+--- SIMILARITY FLAGS ---
+{similarity_flags}
+--- END SIMILARITY FLAGS ---
+
+The response should be a Python-style list of dictionaries matching the required output format.
+
+Run through this 12-point verification checklist and report results:
+
+1. Output Structure
+   - Is the response a list?
+   - Does every list item contain the required fields:
+     - budget_year
+     - budget_value
+     - currency
+     - responsible_institution
+     - evidence
+
+2. Evidence Structure
+   - Is the evidence field a list?
+   - Does every evidence item contain:
+     - quote
+     - source_document
+     - page_or_section
+
+3. Required Field Completeness
+   - Are all extracted fields populated when information is available in the context?
+   - Are values left blank only when the information cannot be found?
+
+4. Context Support
+   - For every extracted budget_year, budget_value, currency, and responsible_institution value, can supporting evidence be found in the provided context?
+
+5. Evidence Alignment
+   - Does each evidence quote directly support the candidate record it is attached to?
+   - Verify that quoted text contains or clearly supports the extracted values.
+
+6. Quote Accuracy
+   - Are all evidence quotes reproduced exactly from the context?
+   - Flag paraphrased, modified, or fabricated quotations.
+
+7. Hallucinated Information
+   - Are any extracted values absent from the context?
+   - Are any responsible institutions, budget values, years, currencies, or program details inferred without evidence?
+
+8. Candidate Coverage
+   - If multiple candidate values are present in the context, were all relevant candidate records extracted?
+   - Flag omitted candidate values that should reasonably have been included.
+
+9. Temporal Coverage
+   - Does the response contain:
+     A. At least one budget allocation occurring five or more years before the current date, if such information exists in the context?
+     B. The most recent budget allocation available in the context?
+   - Flag cases where qualifying records exist but were omitted.
+
+10. Data Consistency
+    - Do budget_year, budget_value, currency, and responsible_institution correspond to the same allocation, project, investment, or financing record?
+    - Flag records that incorrectly combine values from different sources or contexts.
+
+11. Phrase-Level Flags
+    - Are any phrase-level flags confirmed by the context review?
+    - Distinguish between true hallucinations and supported content.
+
+12. Similarity Flags
+    - Do similarity flags indicate unsupported semantic drift between the response and source evidence?
+    - Flag extracted values that do not accurately reflect source meaning.
+
+Severity Guidance:
+- High: hallucinated values, unsupported extractions, incorrect quotes, invalid structure, missing key candidate records, or incorrect temporal selection.
+- Medium: incomplete extraction, weak evidence linkage, inconsistent field population, or partial coverage.
+- Low: formatting issues, minor omissions in evidence metadata, or non-critical organizational issues.
+
+Return your analysis as JSON in the following format:
+
+{{
+  "errors": [
+    {{
+      "check": 1,
+      "description": "...",
+      "severity": "high|medium|low"
+    }}
+  ],
+  "error_count": 0,
+  "pass": true,
+  "summary": "Brief overall assessment"
+}}
+
+Additional Rules:
+- Consider only evidence present in the supplied context.
+- Do not penalize blank values when the information is genuinely unavailable in the context.
+- A candidate record passes verification if every populated field is supported by the attached evidence.
+- If the response contains no valid supporting evidence for a populated field, treat this as a hallucination.
+- If no errors are found, return:
+  - "errors": []
+  - "error_count": 0
+  - "pass": true
+"""
+
+SC_VERIFICATION_PROMPT_TEMPLATE = """\
+--- RESPONSE TO VERIFY ---
+{response}
+--- END RESPONSE ---
+
+--- CONTEXT ---
+{context}
+--- END CONTEXT ---
+
+--- PHRASE-LEVEL FLAGS ---
+{phrase_flags}
+--- END PHRASE FLAGS ---
+
+--- SIMILARITY FLAGS ---
+{similarity_flags}
+--- END SIMILARITY FLAGS ---
+
+The response should be a dictionary containing the following keys:
+
+- development_plan
+- agricultural_strategy
+- standalone_strategy
+- flagship_project
+
+Each key should contain a list of document records.
+
+Run through this 13-point verification checklist and report results:
+
+1. Output Structure
+   - Is the response a dictionary/object?
+   - Does it contain all required top-level keys?
+     - development_plan
+     - agricultural_strategy
+     - standalone_strategy
+     - flagship_project
+
+2. Record Structure
+   - Does every document record contain:
+     - quote
+     - document
+     - page_or_section
+     - year
+
+3. Required Field Completeness
+   - Are all fields populated when information is available in the context?
+   - Are values left blank only when information cannot be found?
+
+4. Context Support
+   - Can every returned document be located in the provided context?
+   - Verify that the document name, publication year, and quote originate from the source material.
+
+5. Quote Accuracy
+   - Are all quotes reproduced exactly from the source?
+   - Flag any paraphrased, altered, truncated, or fabricated quotations.
+
+6. Mention Verification
+   - Does every quote contain or directly support a mention of the target value chain?
+   - Flag documents included without evidence of value-chain relevance.
+
+7. Document Classification Accuracy
+   - Is each document assigned to the correct category?
+
+   Development Plan:
+   - National medium-term development plan.
+   - Country-wide development framework.
+
+   Agricultural Strategy:
+   - Agriculture-specific strategy, investment plan,
+     transformation plan, or equivalent.
+
+   Standalone Strategy:
+   - Strategy primarily dedicated to the target value chain.
+
+   Flagship Project:
+   - Major donor-supported initiative, project,
+     or program focused on the value chain.
+
+   Flag documents assigned to incorrect categories.
+
+8. Duplicate Document Detection
+   - Has the same document been repeated multiple times
+     within a category?
+   - Flag unnecessary duplicates.
+
+9. Coverage Completeness
+   - Were all qualifying documents identified from the context?
+   - Flag relevant documents that were omitted.
+
+10. Recency Prioritization
+    - When multiple qualifying documents exist,
+      does the response appropriately prioritize
+      documents published within the last five years?
+    - Flag if older documents are included while more recent
+      qualifying documents were ignored.
+
+11. Hallucinated Information
+    - Are any document names, years, categories,
+      projects, or quotations absent from the context?
+    - Flag any unsupported additions.
+
+12. Phrase-Level Flags
+    - Are any phrase-level flags confirmed by review of the context?
+    - Distinguish between genuine hallucinations and supported content.
+
+13. Similarity Flags
+    - Do similarity flags indicate semantic drift between
+      extracted information and the source material?
+    - Flag cases where the meaning of the source has been altered.
+
+Severity Guidance:
+
+- High:
+  - Hallucinated documents
+  - Hallucinated quotes
+  - Incorrect category assignment
+  - Unsupported value-chain relevance
+  - Missing major qualifying documents
+  - Invalid output structure
+
+- Medium:
+  - Incomplete document coverage
+  - Missing years
+  - Missing location information
+  - Weak evidence linkage
+  - Failure to prioritize recent documents
+
+- Low:
+  - Duplicate records
+  - Minor formatting issues
+  - Missing non-critical metadata
+
+Return your analysis as JSON in the following format:
+
+{{
+  "errors": [
+    {{
+      "check": 1,
+      "description": "...",
+      "severity": "high|medium|low"
+    }}
+  ],
+  "error_count": 0,
+  "pass": true,
+  "summary": "Brief overall assessment"
+}}
+
+Additional Rules:
+
+- Consider only evidence available in the supplied context.
+- Do not penalize empty categories if no qualifying documents exist in the context.
+- A document record passes verification only if its quote provides direct support for its inclusion.
+- If a document is included without supporting evidence, treat it as a hallucination.
+- If a document clearly belongs in a different category, record a classification error.
+- If no errors are found, return:
+  - "errors": []
+  - "error_count": 0
+  - "pass": true
+"""
+
+IC_VERIFICATION_PROMPT_TEMPLATE = """\
+--- RESPONSE TO VERIFY ---
+{response}
+--- END RESPONSE ---
+
+--- CONTEXT ---
+{context}
+--- END CONTEXT ---
+
+--- PHRASE-LEVEL FLAGS ---
+{phrase_flags}
+--- END PHRASE FLAGS ---
+
+--- SIMILARITY FLAGS ---
+{similarity_flags}
+--- END SIMILARITY FLAGS ---
+
+The response should be a dictionary containing the following keys:
+
+- dedicated_agency
+- dedicated_research_institute
+- dedicated_ministerial_department
+
+Each key should contain a list of organizational records.
+
+Run through this 13-point verification checklist and report results:
+
+1. Output Structure
+   - Is the response a dictionary/object?
+   - Does it contain all required top-level keys?
+     - dedicated_agency
+     - dedicated_research_institute
+     - dedicated_ministerial_department
+
+2. Record Structure
+   - Does every record contain the required fields?
+
+   dedicated_agency:
+     - agency_name
+     - document
+     - page_or_section
+     - quote
+
+   dedicated_research_institute:
+     - institution_name
+     - document
+     - page_or_section
+     - quote
+
+   dedicated_ministerial_department:
+     - department_name
+     - document
+     - page_or_section
+     - quote
+
+3. Required Field Completeness
+   - Are all fields populated when information is available in the context?
+   - Are values left blank only when information cannot be determined from the context?
+
+4. Context Support
+   - Can every organization name be verified from the provided context?
+   - Can the cited document and page/section be located in the context?
+
+5. Quote Accuracy
+   - Are all quotes reproduced exactly from the source material?
+   - Flag paraphrased, altered, incomplete, or fabricated quotations.
+
+6. Organizational Relevance
+   - Does each quote directly indicate that the organization has responsibility for, oversight of, coordination of, regulation of, research on, or support for the target value chain?
+   - Flag organizations included without clear value-chain relevance.
+
+7. Category Assignment Accuracy
+   - Is each organization assigned to the correct category?
+
+   Dedicated Agency:
+   - Government agency, authority, board, corporation, commission,
+     parastatal, or similar public institution dedicated to the value chain.
+
+   Dedicated Research Institute:
+   - Research institute, research center, laboratory,
+     or specialized research organization dedicated to the value chain.
+
+   Dedicated Ministerial Department:
+   - Ministry department, directorate, unit, division,
+     board, council, authority, or administrative office
+     responsible for the value chain.
+
+   Flag organizations assigned to incorrect categories.
+
+8. Evidence Strength
+   - Does the quote provide direct evidence of the organization's mandate,
+     role, or dedicated focus?
+   - Flag records where organizational relevance is inferred
+     without explicit supporting language.
+
+9. Duplicate Entity Detection
+   - Has the same organization been returned multiple times
+     within the same category?
+   - Flag unnecessary duplicate entries.
+
+10. Coverage Completeness
+    - Were all qualifying organizations identified from the context?
+    - Flag relevant agencies, institutes, departments,
+      authorities, boards, councils, or units that were omitted.
+
+11. Hallucinated Information
+    - Are any organization names, document names,
+      quotations, or classifications absent from the context?
+    - Flag unsupported additions.
+
+12. Phrase-Level Flags
+    - Are any phrase-level flags confirmed by review of the context?
+    - Distinguish between genuine hallucinations and supported content.
+
+13. Similarity Flags
+    - Do similarity flags indicate semantic drift between
+      extracted information and the source material?
+    - Flag records where the extracted organizational role
+      differs from what the source actually states.
+
+Severity Guidance:
+
+- High:
+  - Hallucinated organizations
+  - Hallucinated quotes
+  - Unsupported value-chain relevance
+  - Incorrect category assignment
+  - Missing major qualifying organizations
+  - Invalid output structure
+
+- Medium:
+  - Weak supporting evidence
+  - Missing organization names
+  - Missing document locations
+  - Incomplete extraction coverage
+  - Ambiguous classification
+
+- Low:
+  - Duplicate entries
+  - Formatting issues
+  - Minor metadata omissions
+
+Special Verification Rules:
+
+- An organization should only be included if the source provides evidence
+  that it is specifically responsible for, focused on, mandated for,
+  or significantly engaged with the target value chain.
+
+- Do not assume an organization is dedicated to the value chain simply because
+  it operates within the broader agriculture sector.
+
+- General ministries (e.g., Ministry of Agriculture) should not automatically
+  qualify as dedicated ministerial departments unless a specific department,
+  directorate, unit, authority, board, or council connected to the value chain
+  is identified.
+
+- Research institutes should only be included when there is evidence of a
+  dedicated value-chain focus or mandate.
+
+- If the same organization appears in multiple categories, verify that the
+  source material genuinely supports each classification.
+  Otherwise record a classification error.
+
+- Consider only evidence available in the supplied context.
+
+- If no qualifying organizations are present in the context,
+  empty lists are valid and should not be treated as errors.
+
+- If a record lacks supporting evidence, treat it as a hallucination.
+
+Return your analysis as JSON in the following format:
+
+{{
+  "errors": [
+    {{
+      "check": 1,
+      "description": "...",
+      "severity": "high|medium|low"
+    }}
+  ],
+  "error_count": 0,
+  "pass": true,
+  "summary": "Brief overall assessment"
+}}
+
+If no errors are found, return:
+  - "errors": []
+  - "error_count": 0
+  - "pass": true
+"""
 
 VERIFICATION_PROMPT_TEMPLATE = """\
 --- RESPONSE TO VERIFY ---
@@ -516,6 +974,12 @@ VC_EXTRACT = {
     "strategic_commitment":STRATEGIC_COMMITMENT_BLOCK,
     "institutional_commitment":INSTITUTIONAL_COMMITMENT_BLOCK,
 }
+
+VC_VERIFY = {
+    "financial_commitment":FC_VERIFICATION_PROMPT_TEMPLATE,
+    "strategic_commitment":SC_VERIFICATION_PROMPT_TEMPLATE,
+    "institutional_commitment":IC_VERIFICATION_PROMPT_TEMPLATE,
+}
 def build_vc_analyzer_prompt(cfg:Dict,
                              retrieval_result:Dict,
                              value_chain:str,
@@ -653,6 +1117,7 @@ def build_verification_prompt(
     context: str,
     phrase_flags: list[str],
     similarity_flags: list[str],
+    commitment_var:str,
 ) -> str:
     """Build the verification prompt for auditing an AI response.
 
@@ -665,6 +1130,7 @@ def build_verification_prompt(
     Returns:
         Formatted verification prompt string.
     """
+    verification_prompt_template = VC_VERIFY[commitment_var]
     phrase_str = (
         "\n".join(f"- {flag}" for flag in phrase_flags)
         if phrase_flags
@@ -675,7 +1141,7 @@ def build_verification_prompt(
         if similarity_flags
         else "No similarity flags detected."
     )
-    return VERIFICATION_PROMPT_TEMPLATE.format(
+    return verification_prompt_template.format(
         response=response,
         context=context,
         phrase_flags=phrase_str,

src/verifier.py

@@ -440,7 +440,7 @@ def verify_and_respond(
         # Layer 3: LLM-as-verifier
         all_sim_flags = sim_flag_strs + citation_flag_strs
         verification_prompt = build_verification_prompt(
-            response, context, phrase_flag_strs, all_sim_flags,
+            response, context, phrase_flag_strs, all_sim_flags, retrieval_result["var"],
         )
         try:
             raw_verification = generate(
@@ -450,7 +450,7 @@ def verify_and_respond(
                 problems.You are a strict verification agent. Return only JSON.""",
                 verification_prompt,
                 cfg,
-                max_tokens=1024,
+                max_tokens=8192,
             )
         except Exception:
             # Verification LLM failed — skip correction, continue to next iteration
@@ -502,6 +502,7 @@ def verify_and_respond(
             response, context,
             [f["message"] for f in phrase_flags],
             all_sim_final,
+            retrieval_result["var"],
         )
         try:
             vr = parse_verification_result(generate(
@@ -509,7 +510,7 @@ def verify_and_respond(
                 You are a verification agent. Your job is to audit the following
                 AI-generated response against the provided context and flag any
                 problems.You are a strict verification agent. Return only JSON.""",
-                vp, cfg, max_tokens=1024,
+                vp, cfg, max_tokens=8192,
             ))
             any_verification_ran = True
         except Exception: