on going

.gitignore

@@ -10,10 +10,9 @@ __pycache__/
 *.code-workspace
 .DS_Store
 Thumbs.db
-*.log
+conversion_log_*.log
 *.tmp
 faiss_index/
 ConvertedFiles/
 OrigFolder/
-TempFolder/
-*.faiss
+TempFolder/

app.py

@@ -5,21 +5,28 @@ from langchain_community.vectorstores import FAISS
 from langchain_community.document_loaders import PyPDFDirectoryLoader, WebBaseLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_tavily import TavilySearch
 from langchain.tools.retriever import create_retriever_tool
 from langchain.agents import create_tool_calling_agent, AgentExecutor
-from langchain_core.tools import tool
 import gradio as gr
 import numpy as np
+import logging
+from pythia import scenario_simulator
 
-#to run locally
-# load_dotenv()
-os.environ['USER_AGENT'] = os.getenv('USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)')
-# Load secrets from Hugging Face environment
-openai_api_key = os.getenv("OPENAI_API_KEY")
-tavily_api_key = os.getenv("TAVILY_API_KEY")
-if not openai_api_key or not tavily_api_key:
-    raise ValueError("API keys not set in Hugging Face Secrets")
+# Set up logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+# os.environ['USER_AGENT'] = os.getenv('USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)')
+
+# # Verify API keys
+# openai_api_key = os.getenv("OPENAI_API_KEY")
+# tavily_api_key = os.getenv("TAVILY_API_KEY")
+# if not openai_api_key or not tavily_api_key:
+#     logger.error("Missing API keys: OPENAI_API_KEY or TAVILY_API_KEY")
+#     raise ValueError("API keys not set in environment")
 
 def load_web_resources(file='websites.txt'):
     urls = ['https://steveblank.com/']
@@ -31,54 +38,41 @@ def load_web_resources(file='websites.txt'):
         try:
             loader = WebBaseLoader(url)
             web_docs.extend(loader.load())
+            logger.info(f"Loaded web resources from {url}")
         except Exception as e:
-            print(f"Failed to load {url}: {e}")
+            logger.error(f"Failed to load {url}: {e}")
     return web_docs
 
 def build_vectorstore(pdf_dir="ConvertedFiles"):
-    loader = PyPDFDirectoryLoader(pdf_dir)
-    docs = loader.load()
-    web_docs = load_web_resources()
-    all_docs = docs + web_docs
-    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-    splits = splitter.split_documents(all_docs)
-    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
-    vectorstore = FAISS.from_documents(splits, embeddings)
-    vectorstore.save_local("faiss_index")
-    return vectorstore
-
-@tool
-def scenario_simulator(params: dict) -> str:
-    """Simulates product adoption using Monte Carlo. Params: initial_users, growth_rate, time_months, uncertainty."""
     try:
-        initial_users = params.get("initial_users", 1000)
-        growth_rate = params.get("growth_rate", 0.05)  # 5%/month
-        time_months = params.get("time_months", 12)
-        uncertainty = params.get("uncertainty", 0.1)  # 10% variance
-        runs = 1000
-        final_users = []
-        for _ in range(runs):
-            users = initial_users
-            for _ in range(time_months):
-                users *= (1 + np.random.normal(growth_rate, uncertainty))
-            final_users.append(users)
-        mean = np.mean(final_users)
-        std = np.std(final_users)
-        return f"Predicted users after {time_months} months: {mean:.0f} ± {std:.0f} (mean ± std, {runs} runs)."
+        loader = PyPDFDirectoryLoader(pdf_dir)
+        docs = loader.load()
+        logger.info(f"Loaded {len(docs)} documents from {pdf_dir}")
+        web_docs = load_web_resources()
+        all_docs = docs + web_docs
+        splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+        splits = splitter.split_documents(all_docs)
+        logger.info(f"Split into {len(splits)} document chunks")
+        embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+        vectorstore = FAISS.from_documents(splits, embeddings)
+        vectorstore.save_local("faiss_index")
+        logger.info("Saved FAISS index to faiss_index/")
+        return vectorstore
     except Exception as e:
-        return f"Sim error: {e}. Check params: {params}"
+        logger.error(f"Failed to build vectorstore: {e}")
+        raise
 
 def setup_agent(vectorstore):
     retriever_tool = create_retriever_tool(
         vectorstore.as_retriever(search_kwargs={"k": 5}),
         "course_materials_retriever",
-        "Searches PDFs and crawled web resources on optimization/sim/decision tech/project mgmt/lean startup."
+        "Searches course materials on optimization, simulation, decision tech, project management, and lean startup."
     )
-    web_tool = TavilySearchResults(max_results=3)
+    web_tool = TavilySearch(max_results=3)
     tools = [retriever_tool, web_tool, scenario_simulator]
     llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)
     prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are Dr. Sue Cohen's Teaching Assistant, a passionate tutor for Commercializing New Technologies, Leading Organizations to Innovate Smarter, Competing in Emerging Economies, and Strategic Management. Guide users with sharp insights, probing questions, and hints from course materials/web. Be witty (e.g., 'Solid, but don’t optimize for chaos—how’s this tie to diffusion?'), professional, encouraging. Run scenario sims for adoption/scalability queries with clear params. Prioritize course/web data, use web for updates. Be concise, cite sources, never solve outright, redirect if off-topic, prompt next steps."),
+        ("system", "You are Dr. Sue Cohen, a passionate professor of Commercializing New Technologies. Answer in a witty, professional, detailed style (100-200 words for course content), using main points from course materials without citing file names in the main response. Optionally include PDF titles as references at the end (e.g., 'Reference: Lecture_5.pdf'). Always use Tavily/web for recent updates in separate paragraphs, citing URLs. Don’t mix course and web content in the same paragraph. Run scenario sims for adoption/scalability with clear params. Redirect off-topic queries, prompt next steps with a sharp question."),
         ("human", "{input}"),
         MessagesPlaceholder("agent_scratchpad")
     ])
@@ -87,20 +81,47 @@ def setup_agent(vectorstore):
     return executor
 
 def chat(message, history):
-    # Parse query for scenario_simulator if needed
+    # Parse query for scenario_simulator
     if "initial_users" in message:
         params = {}
-        for pair in message.split(","):
-            key, value = pair.split(":")
-            key = key.strip()
-            value = float(value.strip().replace("%", "")) / 100 if "%" in value else int(value.strip())
-            params[key] = value
-        return scenario_simulator(params)
-    return agent.invoke({"input": message})["output"]
+        try:
+            for pair in message.split(","):
+                key, value = pair.split(":")
+                key = key.strip()
+                value = float(value.strip().replace("%", "")) / 100 if "%" in value else int(value.strip())
+                params[key] = value
+            logger.info(f"Running scenario simulator with params: {params}")
+            return scenario_simulator(params)
+        except Exception as e:
+            logger.error(f"Failed to parse scenario params: {e}")
+            return f"Error parsing params: {e}. Format: initial_users:1000, growth_rate:5%, time_months:12, uncertainty:10%"
+    
+    # Retrieve documents
+    retriever = vectorstore.as_retriever(search_kwargs={"k": 5})
+    docs = retriever.get_relevant_documents(message)
+    context = "\n\n".join([doc.page_content for doc in docs])
+    titles = [os.path.basename(doc.metadata.get('source', 'Unknown')) for doc in docs]
+    logger.info(f"Retrieved {len(docs)} documents for query: {message}")
+    
+    # Always invoke Tavily for updates
+    try:
+        web_results = TavilySearch(max_results=3).invoke(message)
+        web_context = "\n".join([f"Web: {res.get('title', 'Untitled')} - {res.get('content', '')[:150]}... (URL: {res.get('url', 'Unknown')})" for res in web_results])
+        logger.info(f"Retrieved {len(web_results)} web results from Tavily")
+    except Exception as e:
+        web_context = ""
+        logger.error(f"Tavily search failed: {e}")
+    
+    # Combine contexts
+    input_with_context = f"{message}\n\nCourse Materials:\n{context}\n\nWeb Updates:\n{web_context}\n\nPDF References:\n{', '.join(titles)}"
+    logger.info(f"Processing query: {message}")
+    return agent.invoke({"input": input_with_context})["output"]
 
 if not os.path.exists("faiss_index"):
+    logger.warning("FAISS index not found, rebuilding...")
     vectorstore = build_vectorstore()
 else:
+    logger.info("Loading prebuilt FAISS index...")
     embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
     vectorstore = FAISS.load_local("faiss_index", embeddings, allow_dangerous_deserialization=True)
 
@@ -108,9 +129,11 @@ agent = setup_agent(vectorstore)
 
 demo = gr.ChatInterface(
     fn=chat,
+    chatbot=gr.Chatbot(type='messages'),
     title="COMMERCIALIZING NEW TECHNOLOGIES RAG Bot",
-    description="Query materials + web augmented"
+    description="Query course materials with web augmentation"
 )
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860,share=True)
+    logger.info("Starting Gradio app...")
+    demo.launch(server_name="0.0.0.0", server_port=7860)

doc_preproc.py

@@ -0,0 +1,124 @@
+import os
+import shutil
+import sys
+import logging
+from datetime import datetime
+
+# Set up logging
+logging.basicConfig(
+    filename=f"conversion_log_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log",
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+# For DOC/DOCX conversion
+try:
+    from docx2pdf import convert as docx_convert
+except ImportError:
+    logging.error("docx2pdf not installed. Run 'pip install docx2pdf'.")
+    sys.exit(1)
+
+# For Word-based conversion
+try:
+    import win32com.client
+except ImportError:
+    logging.error("pywin32 not installed. Run 'pip install pywin32'.")
+    sys.exit(1)
+
+try:
+    import comtypes.client
+except ImportError:
+    logging.warning("comtypes not installed. Run 'pip install comtypes' for fallback.")
+
+def check_irm_status(filepath):
+    """Check if IRM is enabled on a DOC/DOCX file."""
+    try:
+        word_app = win32com.client.Dispatch("Word.Application")
+        doc = word_app.Documents.Open(os.path.abspath(filepath), ReadOnly=True, Visible=False)
+        irm_enabled = doc.Permission.Enabled
+        doc.Close()
+        word_app.Quit()
+        return irm_enabled
+    except Exception as e:
+        logging.error(f"Failed to check IRM for {filepath}: {e}")
+        return None
+
+def convert_docx_docx2pdf(filepath, output_pdf):
+    """Attempt DOC/DOCX to PDF conversion using docx2pdf."""
+    try:
+        docx_convert(filepath, output_pdf)
+        logging.info(f"Converted {filepath} to {output_pdf} using docx2pdf.")
+        return True
+    except Exception as e:
+        logging.error(f"docx2pdf failed for {filepath}: {e}")
+        return False
+
+def convert_docx_win32(filepath, output_pdf):
+    """Attempt DOC/DOCX to PDF conversion using win32com SaveAs."""
+    try:
+        word_app = win32com.client.Dispatch("Word.Application")
+        doc = word_app.Documents.Open(os.path.abspath(filepath), ReadOnly=True, Visible=False)
+        doc.SaveAs(output_pdf, FileFormat=17)  # 17 = wdFormatPDF
+        doc.Close()
+        word_app.Quit()
+        logging.info(f"Converted {filepath} to {output_pdf} using win32com SaveAs.")
+        return True
+    except Exception as e:
+        logging.error(f"win32com SaveAs failed for {filepath}: {e}")
+        return False
+
+def convert_docx_comtypes(filepath, output_pdf):
+    """Attempt DOC/DOCX to PDF conversion using comtypes SaveAs."""
+    try:
+        word_app = comtypes.client.CreateObject("Word.Application")
+        doc = word_app.Documents.Open(os.path.abspath(filepath), ReadOnly=True, Visible=False)
+        doc.SaveAs(output_pdf, FileFormat=17)  # 17 = wdFormatPDF
+        doc.Close()
+        word_app.Quit()
+        logging.info(f"Converted {filepath} to {output_pdf} using comtypes SaveAs.")
+        return True
+    except Exception as e:
+        logging.error(f"comtypes SaveAs failed for {filepath}: {e}")
+        return False
+
+def main(input_dir, output_dir):
+    if not os.path.isdir(input_dir):
+        logging.error(f"{input_dir} is not a directory.")
+        sys.exit(1)
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    for root, _, files in os.walk(input_dir):
+        rel_dir = os.path.relpath(root, input_dir)
+        target_subdir = os.path.join(output_dir, rel_dir)
+        os.makedirs(target_subdir, exist_ok=True)
+
+        for file in files:
+            filepath = os.path.join(root, file)
+            base, ext = os.path.splitext(file)
+            output_pdf = os.path.join(target_subdir, base + '.pdf')
+
+            try:
+                if ext.lower() in ['.doc', '.docx']:
+                    # Check IRM status
+                    irm_status = check_irm_status(filepath)
+                    if irm_status is not None:
+                        logging.info(f"IRM enabled for {filepath}: {irm_status}")
+
+                    # Try docx2pdf first (simpler, faster)
+                    if not convert_docx_docx2pdf(filepath, output_pdf):
+                        # Fallback to win32com
+                        if not convert_docx_win32(filepath, output_pdf):
+                            # Fallback to comtypes if available
+                            if 'comtypes' in sys.modules and not convert_docx_comtypes(filepath, output_pdf):
+                                logging.warning(f"Failed to convert {filepath}. Likely IRM or strict restrictions.")
+                            elif not 'comtypes' in sys.modules:
+                                logging.warning(f"Failed to convert {filepath}. Install comtypes for additional fallback.")
+                elif ext.lower() == '.pdf':
+                    shutil.copy(filepath, output_pdf)
+                    logging.info(f"Copied {filepath} to {output_pdf}")
+            except Exception as e:
+                logging.error(f"Error processing {filepath}: {e}")
+
+if __name__ == "__main__":
+    main("ToAdd", "ConvertedFiles")

requirements.txt

@@ -8,4 +8,9 @@ python-dotenv>=1.0.1
 langchain-text-splitters>=0.3.0
 tavily-python>=0.4.0
 numpy>=1.26.4
-beautifulsoup4>=4.12.0
+beautifulsoup4>=4.12.0
+docx2pdf>=0.2.2
+pywin32>=306
+comtypes>=1.4.7
+pypdf>=5.0.0 
+comtypes>=1.4.7