Update app.py

app.py

@@ -1,62 +1,120 @@
-# DDS Enterprise HR Chatbot + Telegram Bot
-# Hugging Face Spaces app.py
+# ============================================================
+# DDS ENTERPRISE HR CHATBOT
+# Hugging Face + LlamaIndex + Pinecone + OpenAI + Telegram
+# ============================================================
 
-import logging
 import os
 import sys
-import threading
 import time
-
-import gradio as gr
+import logging
+import threading
 import requests
+import gradio as gr
+
 from pinecone import Pinecone, ServerlessSpec
 
 from llama_index.core import (
-    Settings,
+    VectorStoreIndex,
     SimpleDirectoryReader,
     StorageContext,
-    VectorStoreIndex,
+    Settings,
 )
-from llama_index.embeddings.openai import OpenAIEmbedding
-from llama_index.llms.openai import OpenAI
-from llama_index.readers.file import PDFReader
+
 from llama_index.vector_stores.pinecone import PineconeVectorStore
+from llama_index.readers.file import PDFReader
+from llama_index.llms.openai import OpenAI
+from llama_index.embeddings.openai import OpenAIEmbedding
+
+
+# ============================================================
+# 1. LOGGING
+# ============================================================
 
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.INFO
+)
+
+logger = logging.getLogger("DDS-HR")
 
-# ---------------------------------------------------------
-# 1. Logging
-# ---------------------------------------------------------
-logging.basicConfig(stream=sys.stdout, level=logging.INFO)
-logger = logging.getLogger("dds-hr-enterprise")
 
+# ============================================================
+# 2. LOAD HUGGING FACE SECRETS
+# ============================================================
 
-# ---------------------------------------------------------
-# 2. Secrets from Hugging Face
-# ---------------------------------------------------------
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
 TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
 
+
 if not OPENAI_API_KEY:
-    raise ValueError("OPENAI_API_KEY is missing from Hugging Face Secrets.")
+    raise ValueError(
+        "OPENAI_API_KEY is missing from Hugging Face Secrets"
+    )
 
 if not PINECONE_API_KEY:
-    raise ValueError("PINECONE_API_KEY is missing from Hugging Face Secrets.")
+    raise ValueError(
+        "PINECONE_API_KEY is missing from Hugging Face Secrets"
+    )
 
 if not TELEGRAM_BOT_TOKEN:
     logger.warning(
-        "TELEGRAM_BOT_TOKEN is missing. Web chatbot will work, "
-        "but Telegram integration will stay disabled."
+        "TELEGRAM_BOT_TOKEN not found. "
+        "Web chatbot will work but Telegram will not."
     )
 
 
-# ---------------------------------------------------------
-# 3. LlamaIndex / OpenAI settings
-# ---------------------------------------------------------
+# ============================================================
+# 3. SYSTEM PROMPT
+# ============================================================
+
+SYSTEM_PROMPT = """
+You are AYesha, the Decoding Data Science (DDS)
+Enterprise HR Chatbot.
+
+You answer questions ONLY using information contained
+in the latest DDS HR Handbook retrieved for you.
+
+RULES:
+
+1. Only answer questions related to DDS HR policies.
+
+2. Base answers only on retrieved DDS HR Handbook content.
+
+3. Never invent HR policies.
+
+4. Never answer confidential questions such as salary
+   information or private employee information.
+
+5. If the handbook does not contain the answer, say:
+
+"I'm sorry, I can only answer questions about the latest
+DDS HR policies. For confidential or other queries,
+please email connect@decodingdatascience.com."
+
+6. If the question is unrelated to HR, say:
+
+"I'm sorry, I only answer DDS HR policy questions
+as outlined in the handbook."
+
+7. Keep answers concise, professional and easy to understand.
+
+8. Internally verify whether the question is within scope
+   before answering. Do not expose internal reasoning.
+
+Always prioritize the DDS HR Handbook as the source of truth.
+"""
+
+
+# ============================================================
+# 4. LLAMAINDEX + OPENAI SETTINGS
+# ============================================================
+
 Settings.llm = OpenAI(
     model="gpt-4o-mini",
     temperature=0.2,
     api_key=OPENAI_API_KEY,
+    system_prompt=SYSTEM_PROMPT,
 )
 
 Settings.embed_model = OpenAIEmbedding(
@@ -68,42 +126,32 @@ Settings.chunk_size = 600
 Settings.chunk_overlap = 200
 
 
-# ---------------------------------------------------------
-# 4. System prompt
-# ---------------------------------------------------------
-SYSTEM_PROMPT = """
-You are AYesha, the Decoding Data Science (DDS) Enterprise HR Chatbot.
-
-Answer questions exclusively using the latest DDS HR Handbook content supplied
-to the retrieval system.
-
-Rules:
-- Only answer questions directly related to DDS HR policies in the handbook.
-- Do not answer general questions unrelated to DDS HR.
-- Do not provide confidential information such as salary details.
-- If the answer is not supported by the handbook, do not guess.
-- For confidential, unsupported, or out-of-scope requests, respond:
-  "I’m sorry, I can only answer questions about the latest DDS HR policies.
-  For confidential or other queries, please email
-  connect@decodingdatascience.com."
-- Keep responses concise, professional, and easy to understand.
-- Base the final answer only on retrieved handbook information.
-"""
+# ============================================================
+# 5. CONNECT TO PINECONE
+# ============================================================
 
+logger.info("Connecting to Pinecone...")
 
-# ---------------------------------------------------------
-# 5. Pinecone
-# ---------------------------------------------------------
-pc = Pinecone(api_key=PINECONE_API_KEY)
+pc = Pinecone(
+    api_key=PINECONE_API_KEY
+)
 
 INDEX_NAME = "quickstart"
 DIMENSION = 1536
 
-existing_indexes = [idx["name"] for idx in pc.list_indexes()]
-index_was_created = INDEX_NAME not in existing_indexes
 
-if index_was_created:
-    logger.info("Creating Pinecone index '%s'...", INDEX_NAME)
+# ============================================================
+# 6. CREATE INDEX ONLY IF IT DOES NOT EXIST
+# ============================================================
+
+existing_indexes = pc.list_indexes().names()
+
+if INDEX_NAME not in existing_indexes:
+
+    logger.info(
+        "Pinecone index does not exist. Creating it..."
+    )
+
     pc.create_index(
         name=INDEX_NAME,
         dimension=DIMENSION,
@@ -114,249 +162,753 @@ if index_was_created:
         ),
     )
 
-    # Give Pinecone a moment to make the new index ready.
-    time.sleep(5)
+    logger.info(
+        "Waiting for Pinecone index..."
+    )
+
+    time.sleep(10)
+
 else:
+
     logger.info(
-        "Using existing Pinecone index '%s' — NOT deleting/rebuilding it.",
-        INDEX_NAME,
+        "Existing Pinecone index found. "
+        "NOT deleting it."
+    )
+
+
+# ============================================================
+# 7. CONNECT TO INDEX
+# ============================================================
+
+pinecone_index = pc.Index(
+    INDEX_NAME
+)
+
+vector_store = PineconeVectorStore(
+    pinecone_index=pinecone_index
+)
+
+
+# ============================================================
+# 8. CHECK WHETHER DATA IS ALREADY INDEXED
+# ============================================================
+
+try:
+
+    stats = pinecone_index.describe_index_stats()
+
+    try:
+        vector_count = stats["total_vector_count"]
+    except Exception:
+        vector_count = getattr(
+            stats,
+            "total_vector_count",
+            0
+        )
+
+except Exception as e:
+
+    logger.warning(
+        f"Could not read Pinecone stats: {e}"
     )
 
-pinecone_index = pc.Index(INDEX_NAME)
-vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
+    vector_count = 0
+
+
+logger.info(
+    f"Pinecone vector count: {vector_count}"
+)
+
 
+# ============================================================
+# 9. INDEX HR PDF IF DATABASE IS EMPTY
+# ============================================================
 
-# ---------------------------------------------------------
-# 6. Build vectors once, or reconnect to existing vectors
-# ---------------------------------------------------------
-if index_was_created:
-    logger.info("Loading PDFs from Data/ and creating embeddings...")
+if vector_count == 0:
+
+    logger.info(
+        "Pinecone is empty. Loading PDFs from Data folder..."
+    )
 
     documents = SimpleDirectoryReader(
         input_dir="Data",
         required_exts=[".pdf"],
-        file_extractor={".pdf": PDFReader()},
+        file_extractor={
+            ".pdf": PDFReader()
+        }
     ).load_data()
 
+
     if not documents:
-        raise ValueError("No PDF documents were loaded from the 'Data' folder.")
+
+        raise ValueError(
+            "No PDF documents found inside Data folder"
+        )
+
+
+    logger.info(
+        f"Loaded {len(documents)} document sections"
+    )
+
 
     storage_context = StorageContext.from_defaults(
         vector_store=vector_store
     )
 
+
+    logger.info(
+        "Creating embeddings and storing them in Pinecone..."
+    )
+
+
     index = VectorStoreIndex.from_documents(
         documents,
         storage_context=storage_context,
     )
 
-    logger.info("DDS HR handbook indexed successfully.")
+
+    logger.info(
+        "DDS HR Handbook indexed successfully."
+    )
+
 
 else:
-    # Reuse the vectors already stored in Pinecone.
+
+    logger.info(
+        "Using existing Pinecone vectors."
+    )
+
+
     index = VectorStoreIndex.from_vector_store(
         vector_store=vector_store
     )
 
 
-# ---------------------------------------------------------
-# 7. Query engine shared by BOTH Gradio and Telegram
-# ---------------------------------------------------------
+# ============================================================
+# 10. QUERY ENGINE
+# ============================================================
+
 query_engine = index.as_query_engine(
-    system_prompt=SYSTEM_PROMPT,
-    similarity_top_k=4,
+    similarity_top_k=4
 )
 
 
-def query_doc(prompt: str) -> str:
-    """Single HR answering function used by web app and Telegram."""
-    if not prompt or not prompt.strip():
-        return "Please enter an HR policy question."
+# ============================================================
+# 11. ONE FUNCTION USED BY WEB + TELEGRAM
+# ============================================================
+
+def query_doc(question):
+
+    if not question:
+        return "Please enter an HR question."
+
+
+    question = question.strip()
+
+
+    if not question:
+        return "Please enter an HR question."
+
+
+    logger.info(
+        f"HR question received: {question}"
+    )
+
+
+    try:
+
+        response = query_engine.query(
+            question
+        )
+
+        answer = str(response)
+
+        logger.info(
+            "Answer generated successfully."
+        )
+
+        return answer
+
+
+    except Exception as e:
+
+        logger.exception(
+            f"RAG ERROR: {e}"
+        )
+
+        return (
+            "Sorry, I couldn't process your question "
+            "right now. Please try again."
+        )
+
+
+# ============================================================
+# 12. TELEGRAM FUNCTIONS
+# ============================================================
+
+def telegram_url(method):
+
+    return (
+        f"https://api.telegram.org/"
+        f"bot{TELEGRAM_BOT_TOKEN}/"
+        f"{method}"
+    )
+
+
+# ============================================================
+# 13. TEST TELEGRAM CONNECTION
+# ============================================================
+
+def test_telegram_connection():
+
+    if not TELEGRAM_BOT_TOKEN:
+        return False
+
 
     try:
-        response = query_engine.query(prompt.strip())
-        return str(response)
 
-    except Exception as exc:
-        logger.exception("Query error")
-        return "Sorry, I couldn't process your question right now. Please try again."
+        response = requests.get(
+            telegram_url("getMe"),
+            timeout=10
+        )
+
+        response.raise_for_status()
+
+        data = response.json()
+
+
+        if data.get("ok"):
+
+            bot = data["result"]
+
+            logger.info(
+                "===================================="
+            )
+
+            logger.info(
+                "TELEGRAM CONNECTED"
+            )
+
+            logger.info(
+                f"Bot Name: {bot.get('first_name')}"
+            )
+
+            logger.info(
+                f"Bot Username: @{bot.get('username')}"
+            )
+
+            logger.info(
+                "===================================="
+            )
+
+            return True
+
+
+        logger.error(
+            f"Telegram rejected token: {data}"
+        )
+
+        return False
+
 
+    except Exception as e:
+
+        logger.exception(
+            f"Telegram connection error: {e}"
+        )
+
+        return False
 
-# ---------------------------------------------------------
-# 8. Telegram integration
-# ---------------------------------------------------------
-def telegram_api(method: str) -> str:
-    return f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/{method}"
 
+# ============================================================
+# 14. SEND TELEGRAM MESSAGE
+# ============================================================
+
+def send_telegram_message(
+    chat_id,
+    text
+):
 
-def telegram_send_message(chat_id: int, text: str) -> None:
-    """Send answer back to Telegram. Split long answers if required."""
     if not TELEGRAM_BOT_TOKEN:
         return
 
+
     text = str(text)
 
-    # Telegram messages have a size limit; 4000 gives us a safe margin.
-    for start in range(0, len(text), 4000):
-        chunk = text[start:start + 4000]
 
-        response = requests.post(
-            telegram_api("sendMessage"),
-            json={
-                "chat_id": chat_id,
-                "text": chunk,
-            },
-            timeout=20,
+    # Telegram messages have a length limit,
+    # so split long answers.
+    chunks = [
+
+        text[i:i + 4000]
+
+        for i in range(
+            0,
+            len(text),
+            4000
         )
-        response.raise_for_status()
 
+    ]
+
+
+    for chunk in chunks:
+
+        try:
+
+            response = requests.post(
+
+                telegram_url(
+                    "sendMessage"
+                ),
+
+                json={
+                    "chat_id": chat_id,
+                    "text": chunk,
+                },
+
+                timeout=15
+            )
+
+
+            response.raise_for_status()
+
+
+        except Exception as e:
+
+            logger.exception(
+                f"Telegram send error: {e}"
+            )
+
+
+# ============================================================
+# 15. SHOW TYPING...
+# ============================================================
+
+def telegram_typing(chat_id):
 
-def telegram_typing(chat_id: int) -> None:
     try:
+
         requests.post(
-            telegram_api("sendChatAction"),
+
+            telegram_url(
+                "sendChatAction"
+            ),
+
             json={
                 "chat_id": chat_id,
                 "action": "typing",
             },
-            timeout=10,
+
+            timeout=5
         )
+
+
     except Exception:
-        # Typing indicator is optional, so ignore failures.
+
         pass
 
 
-def telegram_polling_loop() -> None:
-    """Receive Telegram messages using long polling."""
+# ============================================================
+# 16. TELEGRAM POLLING
+#
+# IMPORTANT:
+# We deliberately use SHORT POLLING for this demo.
+# This avoids the 55-second Hugging Face timeout you saw.
+# ============================================================
+
+def telegram_polling_loop():
+
     if not TELEGRAM_BOT_TOKEN:
+
+        logger.warning(
+            "Telegram disabled because "
+            "TELEGRAM_BOT_TOKEN is missing."
+        )
+
+        return
+
+
+    # ----------------------------------------------
+    # Verify token first
+    # ----------------------------------------------
+
+    if not test_telegram_connection():
+
+        logger.error(
+            "Telegram connection failed."
+        )
+
         return
 
-    logger.info("Starting DDS HR Telegram bot...")
 
-    # getUpdates and webhooks cannot be used at the same time.
-    # Remove an old webhook if one exists.
+    # ----------------------------------------------
+    # Delete old webhook
+    # getUpdates will not work with a webhook active.
+    # ----------------------------------------------
+
     try:
+
         requests.post(
-            telegram_api("deleteWebhook"),
-            json={"drop_pending_updates": True},
-            timeout=20,
+
+            telegram_url(
+                "deleteWebhook"
+            ),
+
+            json={
+                "drop_pending_updates": True
+            },
+
+            timeout=10
+        )
+
+
+        logger.info(
+            "Old Telegram webhook cleared."
         )
-    except Exception as exc:
-        logger.warning("Could not clear Telegram webhook: %s", exc)
+
+
+    except Exception as e:
+
+        logger.warning(
+            f"Could not clear webhook: {e}"
+        )
+
+
+    logger.info(
+        "===================================="
+    )
+
+    logger.info(
+        "DDS HR TELEGRAM BOT STARTED"
+    )
+
+    logger.info(
+        "Waiting for Telegram messages..."
+    )
+
+    logger.info(
+        "===================================="
+    )
+
 
     offset = None
 
+
     while True:
+
         try:
+
             params = {
-                "timeout": 45,
-                "allowed_updates": ["message"],
+                "timeout": 0
             }
 
+
             if offset is not None:
+
                 params["offset"] = offset
 
+
+            # SHORT POLLING
             response = requests.get(
-                telegram_api("getUpdates"),
+
+                telegram_url(
+                    "getUpdates"
+                ),
+
                 params=params,
-                timeout=55,
+
+                timeout=10
             )
+
+
             response.raise_for_status()
 
-            payload = response.json()
 
-            if not payload.get("ok"):
-                logger.warning("Telegram API response: %s", payload)
-                time.sleep(3)
+            data = response.json()
+
+
+            if not data.get("ok"):
+
+                logger.warning(
+                    f"Telegram API error: {data}"
+                )
+
+                time.sleep(2)
+
                 continue
 
-            for update in payload.get("result", []):
-                offset = update["update_id"] + 1
 
-                message = update.get("message")
+            updates = data.get(
+                "result",
+                []
+            )
+
+
+            for update in updates:
+
+                offset = (
+                    update["update_id"]
+                    + 1
+                )
+
+
+                message = update.get(
+                    "message"
+                )
+
+
                 if not message:
+
                     continue
 
-                chat_id = message.get("chat", {}).get("id")
-                text = (message.get("text") or "").strip()
 
-                if not chat_id or not text:
+                chat = message.get(
+                    "chat",
+                    {}
+                )
+
+
+                chat_id = chat.get(
+                    "id"
+                )
+
+
+                text = (
+                    message.get(
+                        "text"
+                    )
+                    or ""
+                ).strip()
+
+
+                if not chat_id:
+
                     continue
 
-                logger.info("Telegram question received from chat_id=%s", chat_id)
 
-                # Telegram commands
-                if text.startswith("/start"):
-                    telegram_send_message(
+                if not text:
+
+                    send_telegram_message(
+
                         chat_id,
-                        "Welcome to AYesha — DDS Enterprise HR Chatbot.\n\n"
-                        "Ask me a question about the latest DDS HR policies.",
+
+                        "Please send me a text "
+                        "question about DDS HR policies."
+
                     )
+
                     continue
 
-                if text.startswith("/help"):
-                    telegram_send_message(
+
+                logger.info(
+                    f"Telegram message received: {text}"
+                )
+
+
+                # ==================================
+                # /start
+                # ==================================
+
+                if text.startswith(
+                    "/start"
+                ):
+
+                    send_telegram_message(
+
                         chat_id,
-                        "Ask a normal HR question, for example:\n"
+
+                        "Welcome to AYesha — "
+                        "DDS Enterprise HR Chatbot.\n\n"
+                        "Ask me a question about the "
+                        "latest DDS HR policies.\n\n"
+                        "For example:\n"
+                        "What is the annual leave policy?"
+
+                    )
+
+                    continue
+
+
+                # ==================================
+                # /help
+                # ==================================
+
+                if text.startswith(
+                    "/help"
+                ):
+
+                    send_telegram_message(
+
+                        chat_id,
+
+                        "Ask me questions about DDS HR "
+                        "policies.\n\n"
+                        "Examples:\n"
                         "• What is the annual leave policy?\n"
                         "• What is the probation policy?\n"
-                        "• How do I request leave?",
+                        "• How do I request leave?"
+
                     )
+
                     continue
 
+
+                # ==================================
+                # OTHER COMMAND
+                # ==================================
+
                 if text.startswith("/"):
-                    telegram_send_message(
+
+                    send_telegram_message(
+
                         chat_id,
-                        "Please type your HR policy question as a normal message.",
+
+                        "Please type your HR question "
+                        "as a normal message."
+
                     )
+
                     continue
 
-                telegram_typing(chat_id)
 
-                answer = query_doc(text)
-                telegram_send_message(chat_id, answer)
+                # ==================================
+                # NORMAL HR QUESTION
+                # ==================================
+
+                telegram_typing(
+                    chat_id
+                )
+
+
+                answer = query_doc(
+                    text
+                )
+
+
+                send_telegram_message(
+
+                    chat_id,
+                    answer
+
+                )
+
+
+            # Prevent aggressive API calling
+            time.sleep(1)
+
+
+        except requests.exceptions.Timeout:
+
+            # Network timeout is not fatal.
+            # Simply try again.
+            logger.warning(
+                "Telegram temporary timeout. Retrying..."
+            )
+
+            time.sleep(2)
+
+
+        except requests.exceptions.RequestException as e:
+
+            logger.warning(
+                f"Telegram network error: {e}"
+            )
+
+            time.sleep(5)
+
+
+        except Exception as e:
+
+            logger.exception(
+                f"Telegram unexpected error: {e}"
+            )
 
-        except Exception as exc:
-            logger.exception("Telegram polling error: %s", exc)
             time.sleep(5)
 
 
-def start_telegram_bot() -> None:
-    """Run Telegram polling without blocking the Gradio web application."""
+# ============================================================
+# 17. START TELEGRAM IN BACKGROUND
+# ============================================================
+
+def start_telegram_bot():
+
     if not TELEGRAM_BOT_TOKEN:
-        logger.warning("Telegram bot not started: TELEGRAM_BOT_TOKEN missing.")
+
+        logger.warning(
+            "Telegram bot NOT started."
+        )
+
         return
 
-    bot_thread = threading.Thread(
+
+    thread = threading.Thread(
+
         target=telegram_polling_loop,
+
         daemon=True,
-        name="telegram-bot",
+
+        name="DDS-Telegram-Bot"
+
+    )
+
+
+    thread.start()
+
+
+    logger.info(
+        "Telegram background thread started."
     )
-    bot_thread.start()
-    logger.info("Telegram bot background thread started.")
 
 
-# Start Telegram before Gradio blocks the main thread.
+# ============================================================
+# 18. START TELEGRAM
+# ============================================================
+
 start_telegram_bot()
 
 
-# ---------------------------------------------------------
-# 9. Gradio web app
-# ---------------------------------------------------------
+# ============================================================
+# 19. GRADIO WEB APP
+# ============================================================
+
 demo = gr.Interface(
+
     fn=query_doc,
+
     inputs=gr.Textbox(
+
         label="Ask a question about DDS HR policies",
-        placeholder="Example: What is the annual leave policy?",
+
+        placeholder=(
+            "Example: What is the annual leave policy?"
+        )
+
     ),
-    outputs=gr.Textbox(label="Answer"),
+
+    outputs=gr.Textbox(
+        label="Answer"
+    ),
+
     title="DDS Enterprise HR Chatbot",
-    description="Ask questions based on the latest DDS HR Handbook.",
+
+    description=(
+        "Ask questions based on the latest "
+        "DDS HR Handbook."
+    )
+
 )
 
 
+# ============================================================
+# 20. RUN HUGGING FACE APP
+# ============================================================
+
 if __name__ == "__main__":
+
     demo.launch(
+
         server_name="0.0.0.0",
-        server_port=7860,
-    )
+
+        server_port=7860
+
+    )