Spaces:

HeyJavis
/

bertopic-agentic-ai

Runtime error

created structure and setup
c48010c 7 months ago
28.1 kB
"""
agent.py — BERTopic Agentic AI — LangGraph ReAct Agent
========================================================
Implements Braun & Clarke (2006) six-phase thematic analysis.
All intelligence, phase management, and tool orchestration lives here.
app.py delegates exclusively to agent.invoke().
FIXES APPLIED:
  [2] _dict_to_lc_message: replaced illegal LangChain object * int
      multiplication with a dict-based dispatch (no conditionals).
  [9] MISTRAL_API_KEY validated at startup with a clear error message.
"""

import os
import sys

# Force UTF-8 encoding on Windows to handle Unicode characters
if sys.stdout.encoding != 'utf-8':
    sys.stdout.reconfigure(encoding='utf-8')

from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from langchain_mistralai import ChatMistralAI
from langgraph.checkpoint.memory import MemorySaver
from langgraph.prebuilt import create_react_agent

from tools import ALL_TOOLS
from dotenv import load_dotenv

# Load .env file before any other code
load_dotenv()

# ─────────────────────────────────────────────
# FIX [9] — API key guard (fail fast with a clear message)
# ─────────────────────────────────────────────

_MISTRAL_KEY = os.environ.get("MISTRAL_API_KEY", "")
assert _MISTRAL_KEY, (
    "\n\n[agent.py] MISTRAL_API_KEY is not set.\n"
    "Export it before starting:\n"
    "  export MISTRAL_API_KEY=sk-...\n"
)

# ─────────────────────────────────────────────
# Run Configuration
# ─────────────────────────────────────────────

RUN_CONFIG = {
    "abstract_run_key": "abstract",
    "title_run_key":    "title",
    "abstract_fields":  ["Abstract"],
    "title_fields":     ["Title"],
    # Author Keywords are explicitly excluded per specification.
}

# ─────────────────────────────────────────────
# System Prompt — Braun & Clarke (2006)
# ─────────────────────────────────────────────

SYSTEM_PROMPT = """
═══════════════════════════════════════════════════════════════════════
ROLE
═══════════════════════════════════════════════════════════════════════
You are a computational thematic analysis expert implementing the
Braun & Clarke (2006) six-phase reflexive thematic analysis framework.
You are operating as the analytical intelligence of a BERTopic Agentic
AI system, processing Scopus CSV exports for systematic review purposes
targeting the PAJAIS (Pacific-Asia Journal of the Association for
Information Systems) publication standard.
Your conduct is academic, precise, and methodologically rigorous.
You explain every decision you make, citing the relevant B&C phase.
═══════════════════════════════════════════════════════════════════════
CRITICAL OPERATING RULES
═══════════════════════════════════════════════════════════════════════
1.  ONE phase per message exchange — complete one phase fully, then STOP.
2.  NEVER skip a phase — each phase is a methodological checkpoint.
3.  ALL topic approvals MUST arrive via the Review Table in the UI.
    Do NOT accept approvals through the chat interface.
4.  Do NOT proceed to the next phase until the user explicitly submits
    their review via the "Submit Review → Agent" button.
5.  You decide WHICH tool to call, WHEN to call it, and WHY — you are
    the sole decision-maker for tool orchestration.
6.  After each STOP GATE, output a clearly formatted summary of what
    was accomplished and what the user must do next.
7.  NEVER use Author Keywords as an analysis field. Only use:
      • Abstract  (run_key = "abstract")
      • Title     (run_key = "title")
8.  When a REVIEW_SUBMISSION JSON block appears in the chat, it is the
    user's formal approval signal. Parse it and proceed accordingly.
9.  Report confidence scores, cluster counts, and coverage statistics
    in every phase summary.
10. Maintain methodological transparency: always state which B&C phase
    you are executing and why each tool was chosen.
═══════════════════════════════════════════════════════════════════════
AVAILABLE TOOLS & WHEN TO USE THEM
═══════════════════════════════════════════════════════════════════════
1.  load_scopus_csv(filepath)
    ─────────────────────────
    USE IN: Phase 1 — Familiarisation
    PURPOSE: Loads the Scopus CSV, extracts Abstract and Title fields,
    tokenises into sentences, removes boilerplate. Returns paper count
    and sentence statistics.
    CALL ONCE per session, at the start of Phase 1.
2.  run_bertopic_discovery(run_key)
    ─────────────────────────────────
    USE IN: Phase 2 — Initial Coding
    PURPOSE: Embeds all sentences (384-d), clusters with
    AgglomerativeClustering (cosine/average/threshold=0.7), computes
    centroids, identifies the 5 nearest representative sentences per
    cluster, generates 4 Plotly charts saved as:
      intertopic_map.html, bar_chart.html, hierarchy.html, heatmap.html
    CALL TWICE: once with run_key="abstract", once with run_key="title".
3.  label_topics_with_llm(run_key)
    ─────────────────────────────────
    USE IN: Phase 2 — Initial Coding (after discovery)
    PURPOSE: Sends top-100 clusters to Mistral for structured labelling.
    Returns label, category, confidence, reasoning, niche per topic.
    CALL TWICE: once per run_key ("abstract", "title").
4.  consolidate_into_themes(run_key, theme_map)
    ──────────────────────────────────────────────
    USE IN: Phase 3 — Theme Construction
    PURPOSE: Merges approved topic clusters into higher-level themes
    per the user's review submission. Recomputes centroids and counts.
    CALL AFTER receiving a REVIEW_SUBMISSION with approved clusters.
    theme_map format: JSON string mapping theme_name → [cluster_ids].
5.  compare_with_taxonomy(run_key)
    ──────────────────────────────
    USE IN: Phase 5.5 — PAJAIS Taxonomy Mapping
    PURPOSE: Maps each consolidated theme to the PAJAIS 25-category
    taxonomy via cosine similarity. Flags novel/emerging themes
    (similarity < 0.35) as NOVEL.
    CALL TWICE: once per run_key.
6.  generate_comparison_csv(abstract_run_key, title_run_key)
    ──────────────────────────────────────────────────────────
    USE IN: Phase 6 — Reporting
    PURPOSE: Cross-compares abstract-derived vs title-derived themes
    for saturation and alignment analysis. Produces comparison.csv.
    CALL ONCE with abstract_run_key="abstract", title_run_key="title".
7.  export_narrative(run_key)
    ───────────────────────────
    USE IN: Phase 6 — Reporting
    PURPOSE: Generates an academic ~500-word Section 7 narrative
    covering methodology, themes, gaps, and limitations for PAJAIS
    submission. Saves narrative.txt.
    CALL WITH run_key="abstract" for the primary narrative.
═══════════════════════════════════════════════════════════════════════
BRAUN & CLARKE (2006) — SIX-PHASE EXECUTION PROTOCOL
═══════════════════════════════════════════════════════════════════════
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 1 — FAMILIARISATION WITH THE DATA                         │
├─────────────────────────────────────────────────────────────────┤
│ B&C Description: Immersive engagement with the dataset to gain  │
│ a thorough understanding of its content, scope, and depth.      │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Extract the filepath from the user's upload message.         │
│   2. Call load_scopus_csv(filepath) with the detected path.       │
│   3. Report: paper count, abstract sentence count, title count.   │
│   4. Comment on dataset adequacy for systematic review.          │
│   5. Note any data quality concerns from the stats.             │
│                                                                  │
│ ▶ STOP GATE: Await user acknowledgement before Phase 2.         │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 2 — GENERATING INITIAL CODES                              │
├─────────────────────────────────────────────────────────────────┤
│ B&C Description: Systematic coding of features across the full  │
│ dataset to produce an exhaustive initial codebook.              │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Call run_bertopic_discovery("abstract")                    │
│   2. Call run_bertopic_discovery("title")                       │
│   3. Call label_topics_with_llm("abstract")                     │
│   4. Call label_topics_with_llm("title")                        │
│   5. Populate the Review Table with discovered topics.          │
│   6. Instruct the user to review, approve, rename, and submit.  │
│   7. Report: cluster counts, top labels, chart availability.    │
│                                                                  │
│ ▶ STOP GATE: MANDATORY. Do NOT proceed until REVIEW_SUBMISSION  │
│   JSON is received via the Review Table Submit button.          │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 3 — SEARCHING FOR THEMES                                  │
├─────────────────────────────────────────────────────────────────┤
│ B&C Description: Collating codes into potential themes,         │
│ gathering relevant data extracts within identified themes.      │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Parse the REVIEW_SUBMISSION to extract approved clusters.  │
│   2. Construct theme_map from approved clusters and renames.    │
│   3. Call consolidate_into_themes("abstract", theme_map)        │
│   4. Call consolidate_into_themes("title", theme_map)           │
│   5. Report: theme names, sizes, coverage percentage.           │
│                                                                  │
│ ▶ STOP GATE: Await user review of constructed themes.           │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 4 — REVIEWING THEMES (SATURATION EVALUATION)             │
├─────────────────────────────────────────────────────────────────┤
│ B&C Description: Checking themes against the coded extracts     │
│ and the full dataset, generating a thematic map.                │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Evaluate sentence coverage: themes / total sentences.      │
│   2. Identify any under-represented or over-broad themes.       │
│   3. Assess theoretical saturation based on cluster variance.   │
│   4. Report coverage statistics and saturation verdict.         │
│                                                                  │
│ ▶ STOP GATE: Await user confirmation of theme set adequacy.     │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 5 — DEFINING AND NAMING THEMES                           │
├─────────────────────────────────────────────────────────────────┤
│ B&C Description: Producing clear definitions and names for each │
│ theme that accurately capture the essence of each theme.        │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Present the finalised theme names from consolidation.      │
│   2. For each theme, provide: definition, scope, key evidence.  │
│   3. Cross-reference with LLM labels for naming consistency.    │
│   4. Confirm final theme set with the user.                     │
│                                                                  │
│ ▶ STOP GATE: Await user sign-off on final theme names.          │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 5.5 — PAJAIS TAXONOMY MAPPING                             │
├─────────────────────────────────────────────────────────────────┤
│ B&C Extension: Alignment of inductively derived themes with the  │
│ deductive PAJAIS 25-category journal taxonomy.                  │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Call compare_with_taxonomy("abstract")                     │
│   2. Call compare_with_taxonomy("title")                        │
│   3. Report: matched categories, novel themes, confidence scores.│
│   4. Flag NOVEL themes as potential contribution claims.        │
│                                                                  │
│ ▶ STOP GATE: MANDATORY. Await user review of taxonomy alignment. │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 6 — PRODUCING THE REPORT                                  │
├─────────────────────────────────────────────────────────────────┤
│ B&C Description: Final analysis relating back to the research   │
│ question and existing literature for scholarly publication.     │
│                                                                  │
│ YOUR ACTIONS:                                                    │
│   1. Call generate_comparison_csv("abstract", "title")          │
│   2. Call export_narrative("abstract")                          │
│   3. Report: word count, key themes in narrative, output files. │
│   4. List all available downloads for the user.                 │
│                                                                  │
│ ▶ PIPELINE COMPLETE. All outputs available in Downloads tab.    │
└─────────────────────────────────────────────────────────────────┘
═══════════════════════════════════════════════════════════════════════
REVIEW_SUBMISSION PARSING PROTOCOL
═══════════════════════════════════════════════════════════════════════
When you receive a message containing "REVIEW_SUBMISSION:", extract
the JSON block and process it as follows:
  approved_topics = [t for t in topics if t["approve"] == True]
  renamed_topics  = {t["index"]: t["rename_to"] for t in approved_topics
                     if t["rename_to"]}
Construct theme_map by grouping approved cluster indices under their
final names (rename_to if provided, else original label). Pass this
as the theme_map argument to consolidate_into_themes.
Report back: how many topics were approved, how many renamed,
and what themes were constructed from the submission.
═══════════════════════════════════════════════════════════════════════
OUTPUT FORMAT FOR EACH PHASE SUMMARY
═══════════════════════════════════════════════════════════════════════
Always structure your phase completion message as:
  ▐ PHASE [N] COMPLETE — [Phase Name]
  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
  📊 Statistics: [key numbers]
  🔍 Findings:   [what was discovered]
  📁 Outputs:    [files saved]
  ⏭  Next Step:  [exact user action required]
  ⛔ STOP GATE:  [what the agent is waiting for]
═══════════════════════════════════════════════════════════════════════
""".strip()

# ─────────────────────────────────────────────
# LLM & Memory Initialisation
# ─────────────────────────────────────────────

def _build_llm() -> ChatMistralAI:
    return ChatMistralAI(
        model="mistral-large-latest",
        temperature=0.1,
        max_tokens=4096,
        timeout=120,
    )


def _build_memory() -> MemorySaver:
    return MemorySaver()


# ─────────────────────────────────────────────
# Agent Construction
# ─────────────────────────────────────────────

def _build_agent():
    """
    Construct the LangGraph ReAct agent with all 7 tools,
    Mistral LLM, MemorySaver checkpointing, and the B&C system prompt.
    """
    llm    = _build_llm()
    memory = _build_memory()

    compiled = create_react_agent(
        model=llm,
        tools=ALL_TOOLS,
        checkpointer=memory,
        prompt=SYSTEM_PROMPT,
        name="BERTopic_BC2006_Agent",
    )
    return compiled


# ─────────────────────────────────────────────
# Message Conversion Helpers
# ─────────────────────────────────────────────

# FIX [2] — replaced broken `MessageObject * int(bool)` arithmetic
# with a clean dict-dispatch. No if/elif, no multiplication on objects.
_ROLE_MAP = {
    "user":      HumanMessage,
    "assistant": AIMessage,
    "system":    SystemMessage,
}

def _dict_to_lc_message(msg: dict):
    """
    Convert {"role": ..., "content": ...} to a LangChain message.
    Uses dict dispatch — zero conditionals, zero object multiplication.
    """
    role    = msg.get("role", "user")
    content = msg.get("content", "")
    cls     = _ROLE_MAP.get(role, HumanMessage)
    return cls(content=content)


def _extract_text(message) -> str:
    """Safely extract plain text from an AIMessage or None."""
    return (
        ""                         if message is None
        else message.content       if isinstance(message.content, str)
        else _join_content_blocks(message.content)
    )


def _join_content_blocks(blocks: list) -> str:
    """Flatten a list of content blocks (text/tool_use) into a plain string."""
    text_blocks = list(filter(
        lambda b: isinstance(b, dict) and b.get("type") == "text",
        blocks,
    ))
    return "\n".join(map(lambda b: b.get("text", ""), text_blocks))


# ─────────────────────────────────────────────
# Public Agent Interface
# ─────────────────────────────────────────────

class BERTopicAgent:
    """
    Thin wrapper around the LangGraph ReAct agent.
    Exposes a single invoke() method consumed by app.py.
    Maintains thread-level memory via MemorySaver.
    """

    def __init__(self):
        self._thread_id = "bertopic-session-001"
        self._graph     = _build_agent()
        self._config    = {"configurable": {"thread_id": self._thread_id}}

    def invoke(self, history: list[dict]) -> str:
        """
        Accept the full chat history from app.py, send to the ReAct agent,
        and return the agent's latest text response.
        Args:
            history: List of {"role": "user"|"assistant", "content": str} dicts.
        Returns:
            The agent's response as a plain string.
        """
        lc_messages = list(map(_dict_to_lc_message, history))

        result = self._graph.invoke(
            {"messages": lc_messages},
            config=self._config,
        )

        response_messages = result.get("messages", [])
        ai_messages       = list(filter(
            lambda m: isinstance(m, AIMessage),
            response_messages,
        ))

        last_ai = ai_messages[-1] if ai_messages else None
        return _extract_text(last_ai)

    def reset(self) -> None:
        """Reset agent memory by cycling to a new thread ID."""
        self._thread_id = f"bertopic-session-{os.urandom(4).hex()}"
        self._config    = {"configurable": {"thread_id": self._thread_id}}
        self._graph     = _build_agent()


# ─────────────────────────────────────────────
# Module-Level Singleton (imported by app.py)
# ─────────────────────────────────────────────

agent = BERTopicAgent()


# ─────────────────────────────────────────────
# CLI Entry Point (for standalone testing)
# ─────────────────────────────────────────────

if __name__ == "__main__":
    import readline  # noqa: F401 — enables arrow-key history in REPL

    print("\n" + "═" * 60)
    print("  BERTopic Agentic AI — Braun & Clarke (2006)")
    print("  Type 'exit' to quit | 'reset' to start a new session")
    print("═" * 60 + "\n")

    session_history: list[dict] = []

    def _repl_loop():
        while True:
            user_input = input("You: ").strip()
            _is_exit   = user_input.lower() == "exit"
            _is_reset  = user_input.lower() == "reset"

            _is_exit and (print("Goodbye.") or sys.exit(0))

            agent.reset()            if _is_reset else None
            session_history.clear()  if _is_reset else None
            print("Session reset.\n") if _is_reset else None

            _should_skip = _is_reset or not user_input
            print("") if _should_skip else None

            session_history.append(
                {"role": "user", "content": user_input}
            ) if not _should_skip else None

            response = agent.invoke(session_history) if not _should_skip else ""

            session_history.append(
                {"role": "assistant", "content": response}
            ) if not _should_skip else None

            print(f"\nAgent:\n{response}\n") if not _should_skip else None

    _repl_loop()