Update app.py

app.py

@@ -1,120 +1,52 @@
-# ============================================================
-# DDS ENTERPRISE HR CHATBOT
-# Hugging Face + LlamaIndex + Pinecone + OpenAI + Telegram
-# ============================================================
+# DDS Enterprise HR Chatbot
+# Hugging Face Spaces app.py
 
+import logging
 import os
 import sys
 import time
-import logging
-import threading
-import requests
-import gradio as gr
 
+import gradio as gr
 from pinecone import Pinecone, ServerlessSpec
 
 from llama_index.core import (
-    VectorStoreIndex,
+    Settings,
     SimpleDirectoryReader,
     StorageContext,
-    Settings,
+    VectorStoreIndex,
 )
-
-from llama_index.vector_stores.pinecone import PineconeVectorStore
-from llama_index.readers.file import PDFReader
-from llama_index.llms.openai import OpenAI
 from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.llms.openai import OpenAI
+from llama_index.readers.file import PDFReader
+from llama_index.vector_stores.pinecone import PineconeVectorStore
 
 
-# ============================================================
-# 1. LOGGING
-# ============================================================
-
-logging.basicConfig(
-    stream=sys.stdout,
-    level=logging.INFO
-)
-
-logger = logging.getLogger("DDS-HR")
-
+# ---------------------------------------------------------
+# 1. Logging
+# ---------------------------------------------------------
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger("dds-hr-enterprise")
 
-# ============================================================
-# 2. LOAD HUGGING FACE SECRETS
-# ============================================================
 
+# ---------------------------------------------------------
+# 2. Secrets from Hugging Face
+# ---------------------------------------------------------
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
-TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
-
 
 if not OPENAI_API_KEY:
-    raise ValueError(
-        "OPENAI_API_KEY is missing from Hugging Face Secrets"
-    )
+    raise ValueError("OPENAI_API_KEY is missing from Hugging Face Secrets.")
 
 if not PINECONE_API_KEY:
-    raise ValueError(
-        "PINECONE_API_KEY is missing from Hugging Face Secrets"
-    )
-
-if not TELEGRAM_BOT_TOKEN:
-    logger.warning(
-        "TELEGRAM_BOT_TOKEN not found. "
-        "Web chatbot will work but Telegram will not."
-    )
-
-
-# ============================================================
-# 3. SYSTEM PROMPT
-# ============================================================
-
-SYSTEM_PROMPT = """
-You are AYesha, the Decoding Data Science (DDS)
-Enterprise HR Chatbot.
-
-You answer questions ONLY using information contained
-in the latest DDS HR Handbook retrieved for you.
-
-RULES:
-
-1. Only answer questions related to DDS HR policies.
-
-2. Base answers only on retrieved DDS HR Handbook content.
-
-3. Never invent HR policies.
-
-4. Never answer confidential questions such as salary
-   information or private employee information.
-
-5. If the handbook does not contain the answer, say:
-
-"I'm sorry, I can only answer questions about the latest
-DDS HR policies. For confidential or other queries,
-please email connect@decodingdatascience.com."
-
-6. If the question is unrelated to HR, say:
-
-"I'm sorry, I only answer DDS HR policy questions
-as outlined in the handbook."
-
-7. Keep answers concise, professional and easy to understand.
-
-8. Internally verify whether the question is within scope
-   before answering. Do not expose internal reasoning.
-
-Always prioritize the DDS HR Handbook as the source of truth.
-"""
-
-
-# ============================================================
-# 4. LLAMAINDEX + OPENAI SETTINGS
-# ============================================================
+    raise ValueError("PINECONE_API_KEY is missing from Hugging Face Secrets.")
 
+# ---------------------------------------------------------
+# 3. LlamaIndex / OpenAI settings
+# ---------------------------------------------------------
 Settings.llm = OpenAI(
     model="gpt-4o-mini",
     temperature=0.2,
     api_key=OPENAI_API_KEY,
-    system_prompt=SYSTEM_PROMPT,
 )
 
 Settings.embed_model = OpenAIEmbedding(
@@ -126,32 +58,42 @@ Settings.chunk_size = 600
 Settings.chunk_overlap = 200
 
 
-# ============================================================
-# 5. CONNECT TO PINECONE
-# ============================================================
+# ---------------------------------------------------------
+# 4. System prompt
+# ---------------------------------------------------------
+SYSTEM_PROMPT = """
+You are AYesha, the Decoding Data Science (DDS) Enterprise HR Chatbot.
+
+Answer questions exclusively using the latest DDS HR Handbook content supplied
+to the retrieval system.
+
+Rules:
+- Only answer questions directly related to DDS HR policies in the handbook.
+- Do not answer general questions unrelated to DDS HR.
+- Do not provide confidential information such as salary details.
+- If the answer is not supported by the handbook, do not guess.
+- For confidential, unsupported, or out-of-scope requests, respond:
+  "I’m sorry, I can only answer questions about the latest DDS HR policies.
+  For confidential or other queries, please email
+  connect@decodingdatascience.com."
+- Keep responses concise, professional, and easy to understand.
+- Base the final answer only on retrieved handbook information.
+"""
 
-logger.info("Connecting to Pinecone...")
 
-pc = Pinecone(
-    api_key=PINECONE_API_KEY
-)
+# ---------------------------------------------------------
+# 5. Pinecone
+# ---------------------------------------------------------
+pc = Pinecone(api_key=PINECONE_API_KEY)
 
 INDEX_NAME = "quickstart"
 DIMENSION = 1536
 
+existing_indexes = [idx["name"] for idx in pc.list_indexes()]
+index_was_created = INDEX_NAME not in existing_indexes
 
-# ============================================================
-# 6. CREATE INDEX ONLY IF IT DOES NOT EXIST
-# ============================================================
-
-existing_indexes = pc.list_indexes().names()
-
-if INDEX_NAME not in existing_indexes:
-
-    logger.info(
-        "Pinecone index does not exist. Creating it..."
-    )
-
+if index_was_created:
+    logger.info("Creating Pinecone index '%s'...", INDEX_NAME)
     pc.create_index(
         name=INDEX_NAME,
         dimension=DIMENSION,
@@ -162,687 +104,102 @@ if INDEX_NAME not in existing_indexes:
         ),
     )
 
-    logger.info(
-        "Waiting for Pinecone index..."
-    )
-
-    time.sleep(10)
-
+    # Give Pinecone a moment to make the new index ready.
+    time.sleep(5)
 else:
-
     logger.info(
-        "Existing Pinecone index found. NOT deleting it."
+        "Using existing Pinecone index '%s' — NOT deleting/rebuilding it.",
+        INDEX_NAME,
     )
 
+pinecone_index = pc.Index(INDEX_NAME)
+vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
 
-# ============================================================
-# 7. CONNECT TO INDEX
-# ============================================================
-
-pinecone_index = pc.Index(
-    INDEX_NAME
-)
-
-vector_store = PineconeVectorStore(
-    pinecone_index=pinecone_index
-)
-
-
-# ============================================================
-# 8. CHECK WHETHER DATA IS ALREADY INDEXED
-# ============================================================
-
-try:
-
-    stats = pinecone_index.describe_index_stats()
-
-    try:
-        vector_count = stats["total_vector_count"]
-    except Exception:
-        vector_count = getattr(
-            stats,
-            "total_vector_count",
-            0
-        )
-
-except Exception as e:
-
-    logger.warning(
-        f"Could not read Pinecone stats: {e}"
-    )
-
-    vector_count = 0
-
-
-logger.info(
-    f"Pinecone vector count: {vector_count}"
-)
-
-
-# ============================================================
-# 9. INDEX HR PDF IF DATABASE IS EMPTY
-# ============================================================
 
-if vector_count == 0:
-
-    logger.info(
-        "Pinecone is empty. Loading PDFs from Data folder..."
-    )
+# ---------------------------------------------------------
+# 6. Build vectors once, or reconnect to existing vectors
+# ---------------------------------------------------------
+if index_was_created:
+    logger.info("Loading PDFs from Data/ and creating embeddings...")
 
     documents = SimpleDirectoryReader(
         input_dir="Data",
         required_exts=[".pdf"],
-        file_extractor={
-            ".pdf": PDFReader()
-        }
+        file_extractor={".pdf": PDFReader()},
     ).load_data()
 
     if not documents:
-
-        raise ValueError(
-            "No PDF documents found inside Data folder"
-        )
-
-    logger.info(
-        f"Loaded {len(documents)} document sections"
-    )
+        raise ValueError("No PDF documents were loaded from the 'Data' folder.")
 
     storage_context = StorageContext.from_defaults(
         vector_store=vector_store
     )
 
-    logger.info(
-        "Creating embeddings and storing them in Pinecone..."
-    )
-
     index = VectorStoreIndex.from_documents(
         documents,
         storage_context=storage_context,
     )
 
-    logger.info(
-        "DDS HR Handbook indexed successfully."
-    )
+    logger.info("DDS HR handbook indexed successfully.")
 
 else:
-
-    logger.info(
-        "Using existing Pinecone vectors."
-    )
-
+    # Reuse the vectors already stored in Pinecone.
     index = VectorStoreIndex.from_vector_store(
         vector_store=vector_store
     )
 
 
-# ============================================================
-# 10. QUERY ENGINE
-# ============================================================
-
+# ---------------------------------------------------------
+# 7. Shared HR query engine
+# ---------------------------------------------------------
 query_engine = index.as_query_engine(
-    similarity_top_k=4
+    system_prompt=SYSTEM_PROMPT,
+    similarity_top_k=4,
 )
 
 
-# ============================================================
-# 11. ONE FUNCTION USED BY WEB + TELEGRAM
-# ============================================================
-
-def query_doc(question):
-
-    if not question:
-        return "Please enter an HR question."
-
-    question = question.strip()
-
-    if not question:
-        return "Please enter an HR question."
-
-    logger.info(
-        f"HR question received: {question}"
-    )
-
-    try:
-
-        response = query_engine.query(
-            question
-        )
-
-        answer = str(response)
-
-        logger.info(
-            "Answer generated successfully."
-        )
-
-        return answer
-
-    except Exception as e:
-
-        logger.exception(
-            f"RAG ERROR: {e}"
-        )
-
-        return (
-            "Sorry, I couldn't process your question "
-            "right now. Please try again."
-        )
-
-
-# ============================================================
-# 12. TELEGRAM URL
-# ============================================================
-
-def telegram_url(method):
-
-    return (
-        f"https://api.telegram.org/"
-        f"bot{TELEGRAM_BOT_TOKEN}/"
-        f"{method}"
-    )
-
-
-# ============================================================
-# 13. TEST TELEGRAM CONNECTION FROM HUGGING FACE
-# ============================================================
-
-def test_telegram_connection():
-
-    if not TELEGRAM_BOT_TOKEN:
-
-        logger.error(
-            "TELEGRAM TEST: Token is missing."
-        )
-
-        return False
-
-    logger.info(
-        "===================================="
-    )
-
-    logger.info(
-        "TESTING HUGGING FACE -> TELEGRAM"
-    )
-
-    logger.info(
-        f"TELEGRAM TOKEN FOUND: {bool(TELEGRAM_BOT_TOKEN)}"
-    )
+def query_doc(prompt: str) -> str:
+    """Single HR answering function used by the web UI and API clients."""
+    if not prompt or not prompt.strip():
+        return "Please enter an HR policy question."
 
     try:
-
-        response = requests.get(
-            telegram_url("getMe"),
-            timeout=30
-        )
-
-        logger.info(
-            f"TELEGRAM TEST STATUS: {response.status_code}"
-        )
-
-        response.raise_for_status()
-
-        data = response.json()
-
-        if data.get("ok"):
-
-            bot = data["result"]
-
-            logger.info(
-                "TELEGRAM TEST RESULT: SUCCESS"
-            )
-
-            logger.info(
-                f"Bot Name: {bot.get('first_name')}"
-            )
-
-            logger.info(
-                f"Bot Username: @{bot.get('username')}"
-            )
-
-            logger.info(
-                "===================================="
-            )
-
-            return True
-
-        logger.error(
-            f"TELEGRAM TEST RESULT: API ERROR: {data}"
-        )
-
-        return False
-
-    except requests.exceptions.Timeout as e:
-
-        logger.error(
-            f"TELEGRAM TEST FAILED: TIMEOUT: {repr(e)}"
-        )
-
-        return False
-
-    except requests.exceptions.RequestException as e:
-
-        logger.error(
-            f"TELEGRAM TEST FAILED: NETWORK ERROR: {repr(e)}"
-        )
-
-        return False
-
-    except Exception as e:
-
-        logger.exception(
-            f"TELEGRAM TEST FAILED: {repr(e)}"
-        )
-
-        return False
-
-
-# ============================================================
-# 14. SEND TELEGRAM MESSAGE
-# ============================================================
-
-def send_telegram_message(chat_id, text):
-
-    if not TELEGRAM_BOT_TOKEN:
-        return
-
-    text = str(text)
-
-    # Telegram messages have a length limit.
-    # Split long answers into smaller pieces.
-    chunks = [
-        text[i:i + 4000]
-        for i in range(0, len(text), 4000)
-    ]
-
-    for chunk in chunks:
-
-        try:
-
-            response = requests.post(
-                telegram_url("sendMessage"),
-                json={
-                    "chat_id": chat_id,
-                    "text": chunk,
-                },
-                timeout=30
-            )
-
-            response.raise_for_status()
-
-        except Exception as e:
-
-            logger.exception(
-                f"Telegram send error: {e}"
-            )
-
-
-# ============================================================
-# 15. SHOW TELEGRAM TYPING
-# ============================================================
-
-def telegram_typing(chat_id):
-
-    try:
-
-        requests.post(
-            telegram_url("sendChatAction"),
-            json={
-                "chat_id": chat_id,
-                "action": "typing",
-            },
-            timeout=10
-        )
-
-    except Exception:
-
-        pass
-
-
-# ============================================================
-# 16. TELEGRAM POLLING LOOP
-# ============================================================
-
-def telegram_polling_loop():
-
-    if not TELEGRAM_BOT_TOKEN:
-
-        logger.warning(
-            "Telegram disabled because "
-            "TELEGRAM_BOT_TOKEN is missing."
-        )
-
-        return
-
-    # --------------------------------------------------------
-    # STEP 1: TEST HUGGING FACE -> TELEGRAM
-    # --------------------------------------------------------
-
-    if not test_telegram_connection():
-
-        logger.error(
-            "Telegram connection failed."
-        )
-
-        logger.error(
-            "Web chatbot will continue running normally."
-        )
-
-        return
-
-
-    # --------------------------------------------------------
-    # STEP 2: DELETE ANY OLD WEBHOOK
-    # --------------------------------------------------------
-
-    try:
-
-        response = requests.post(
-            telegram_url("deleteWebhook"),
-            json={
-                "drop_pending_updates": True
-            },
-            timeout=30
-        )
-
-        response.raise_for_status()
-
-        logger.info(
-            "Old Telegram webhook cleared."
-        )
-
-    except Exception as e:
-
-        logger.warning(
-            f"Could not clear webhook: {e}"
-        )
-
-
-    # --------------------------------------------------------
-    # STEP 3: START LISTENING FOR TELEGRAM MESSAGES
-    # --------------------------------------------------------
-
-    logger.info(
-        "===================================="
-    )
-
-    logger.info(
-        "DDS HR TELEGRAM BOT STARTED"
-    )
-
-    logger.info(
-        "Waiting for Telegram messages..."
-    )
-
-    logger.info(
-        "===================================="
-    )
-
-
-    offset = None
-
-
-    while True:
-
-        try:
-
-            params = {
-                "timeout": 0
-            }
-
-            if offset is not None:
-                params["offset"] = offset
-
-
-            response = requests.get(
-                telegram_url("getUpdates"),
-                params=params,
-                timeout=30
-            )
-
-            response.raise_for_status()
-
-            data = response.json()
-
-
-            if not data.get("ok"):
-
-                logger.warning(
-                    f"Telegram API error: {data}"
-                )
-
-                time.sleep(2)
-
-                continue
-
-
-            updates = data.get(
-                "result",
-                []
-            )
-
-
-            for update in updates:
-
-                offset = (
-                    update["update_id"] + 1
-                )
-
-                message = update.get(
-                    "message"
-                )
-
-                if not message:
-                    continue
-
-
-                chat = message.get(
-                    "chat",
-                    {}
-                )
-
-                chat_id = chat.get(
-                    "id"
-                )
-
-                text = (
-                    message.get(
-                        "text"
-                    )
-                    or ""
-                ).strip()
-
-
-                if not chat_id:
-                    continue
-
-
-                if not text:
-
-                    send_telegram_message(
-                        chat_id,
-                        "Please send me a text "
-                        "question about DDS HR policies."
-                    )
-
-                    continue
-
-
-                logger.info(
-                    f"Telegram message received: {text}"
-                )
-
-
-                # ============================================
-                # /start
-                # ============================================
-
-                if text.startswith("/start"):
-
-                    send_telegram_message(
-                        chat_id,
-                        "Welcome to AYesha — "
-                        "DDS Enterprise HR Chatbot.\n\n"
-                        "Ask me a question about the "
-                        "latest DDS HR policies.\n\n"
-                        "For example:\n"
-                        "What is the annual leave policy?"
-                    )
-
-                    continue
-
-
-                # ============================================
-                # /help
-                # ============================================
-
-                if text.startswith("/help"):
-
-                    send_telegram_message(
-                        chat_id,
-                        "Ask me questions about DDS HR "
-                        "policies.\n\n"
-                        "Examples:\n"
-                        "• What is the annual leave policy?\n"
-                        "• What is the probation policy?\n"
-                        "• How do I request leave?"
-                    )
-
-                    continue
-
-
-                # ============================================
-                # OTHER COMMAND
-                # ============================================
-
-                if text.startswith("/"):
-
-                    send_telegram_message(
-                        chat_id,
-                        "Please type your HR question "
-                        "as a normal message."
-                    )
-
-                    continue
-
-
-                # ============================================
-                # NORMAL HR QUESTION
-                # ============================================
-
-                telegram_typing(
-                    chat_id
-                )
-
-                answer = query_doc(
-                    text
-                )
-
-                send_telegram_message(
-                    chat_id,
-                    answer
-                )
-
-
-            # Prevent aggressive Telegram API calls
-            time.sleep(1)
-
-
-        except requests.exceptions.Timeout:
-
-            logger.warning(
-                "Telegram temporary timeout. Retrying..."
-            )
-
-            time.sleep(2)
-
-
-        except requests.exceptions.RequestException as e:
-
-            logger.warning(
-                f"Telegram network error: {e}"
-            )
-
-            time.sleep(5)
-
-
-        except Exception as e:
-
-            logger.exception(
-                f"Telegram unexpected error: {e}"
-            )
-
-            time.sleep(5)
-
-
-# ============================================================
-# 17. START TELEGRAM IN BACKGROUND
-# ============================================================
-
-def start_telegram_bot():
-
-    if not TELEGRAM_BOT_TOKEN:
-
-        logger.warning(
-            "Telegram bot NOT started."
-        )
-
-        return
-
-    thread = threading.Thread(
-        target=telegram_polling_loop,
-        daemon=True,
-        name="DDS-Telegram-Bot"
-    )
-
-    thread.start()
-
-    logger.info(
-        "Telegram background thread started."
-    )
-
-
-# ============================================================
-# 18. START TELEGRAM
-# ============================================================
-
-start_telegram_bot()
-
-
-# ============================================================
-# 19. GRADIO WEB APP
-# ============================================================
+        response = query_engine.query(prompt.strip())
+        return str(response)
+
+    except Exception as exc:
+        logger.exception("Query error")
+        return "Sorry, I couldn't process your question right now. Please try again."
+
+
+# ---------------------------------------------------------
+# 8. Gradio web app + API
+# ---------------------------------------------------------
+# IMPORTANT:
+# - The browser UI still uses query_doc directly.
+# - External clients (Make, Telegram, etc.) can call api_name="/query_doc".
+# - query_doc returns ONLY the answer string. There is no Telegram polling,
+#   webhook handling, token, or Telegram network call in this Space.
+#
+# Gradio's low-level /call endpoint uses an event stream internally.
+# For the simplest external integration, use a Gradio client/API action that
+# returns the prediction value rather than manually parsing the SSE wrapper.
 
 demo = gr.Interface(
-
     fn=query_doc,
-
     inputs=gr.Textbox(
         label="Ask a question about DDS HR policies",
-        placeholder=(
-            "Example: What is the annual leave policy?"
-        )
+        placeholder="Example: What is the annual leave policy?",
     ),
-
-    outputs=gr.Textbox(
-        label="Answer"
-    ),
-
+    outputs=gr.Textbox(label="Answer"),
     title="DDS Enterprise HR Chatbot",
-
-    description=(
-        "Ask questions based on the latest "
-        "DDS HR Handbook."
-    )
-
+    description="Ask questions based on the latest DDS HR Handbook.",
+    api_name="query_doc",
 )
 
 
-# ============================================================
-# 20. RUN HUGGING FACE APP
-# ============================================================
-
 if __name__ == "__main__":
-
     demo.launch(
         server_name="0.0.0.0",
-        server_port=7860
-    )
+        server_port=7860,
+    )