Removed unnecessary dependencies

Dockerfile

@@ -2,40 +2,29 @@ FROM python:3.11-slim
 
 # Set Hugging Face home to persistent storage
 ENV HF_HOME=/data/.huggingface
-
-# Create expected directory structure
-RUN mkdir -p /data/processed_data
-
 WORKDIR /app
 
-# Install system dependencies for scipy
-RUN apt-get update && apt-get install -y \
-    build-essential \
+# Install system dependencies first
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    python3-dev \
+    libgomp1 \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy requirements first for better caching
+# Copy and install Python dependencies
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Explicitly install sentence-transformers (sometimes needed for HF Spaces)
-RUN pip install --no-cache-dir sentence-transformers==2.3.0
+# Create directory structure for Hugging Face Spaces
+RUN mkdir -p /data/processed_data && chmod -R 777 /data
 
-# Create necessary directories
-RUN mkdir -p data/processed_data
-
-# Copy the application code
+# Copy application code
 COPY app/ ./app/
-COPY data/processed_data/chunks.pkl ./data/processed_data/
-COPY data/processed_data/embedded_docs.pkl ./data/processed_data/
 COPY check_dependencies.py ./
 
-# Enable more verbose logging
-ENV PYTHONUNBUFFERED=1
-
-# Run a quick check to verify dependencies are installed correctly
-RUN python check_dependencies.py || echo "Dependency check completed with warnings"
+# Verify dependencies
+RUN python check_dependencies.py
 
-# Set the entry point to run the Streamlit app
-# Use debug.py to troubleshoot if the main app fails
+# Expose and run
 EXPOSE 8501
-CMD ["streamlit", "run", "app/app.py", "--server.address=0.0.0.0", "--server.port=8501", "--logger.level=debug"] 
+CMD ["streamlit", "run", "app/app.py", "--server.address=0.0.0.0", "--server.port=8501"]

check_dependencies.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
+
 """
 Script to check installed dependencies for troubleshooting
 """
+
 import sys
 import pkg_resources
 import importlib
@@ -11,7 +13,8 @@ def check_imports():
     """Check if critical imports work"""
     success = True
     critical_imports = [
-        "sentence_transformers",
+        "transformers",
+        "torch", 
         "streamlit",
         "langchain_core",
         "langchain_openai",
@@ -19,7 +22,7 @@ def check_imports():
         "numpy",
         "scipy"
     ]
-    
+
     print("\n=== Import Checks ===")
     for module_name in critical_imports:
         try:
@@ -31,21 +34,35 @@ def check_imports():
             print(f"✗ {module_name}: FAILED - {str(e)}")
     return success
 
-def check_sentence_transformer():
-    """Check if SentenceTransformer works properly"""
-    print("\n=== SentenceTransformer Check ===")
+def check_transformers_model():
+    """Check if transformers can load your fine-tuned model"""
+    print("\n=== Transformers Model Check ===")
+    try:
+        from transformers import AutoModel, AutoTokenizer
+        model_name = "kamkol/ab_testing_finetuned_arctic_ft-36dfff22-0696-40d2-b3bf-268fe2ff2aec"
+        print(f"Testing loading model: {model_name}")
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModel.from_pretrained(model_name)
+        inputs = tokenizer("Test sentence", return_tensors="pt")
+        with torch.no_grad():
+            outputs = model(**inputs)
+        print("✓ Successfully loaded model and ran forward pass")
+        return True
+    except Exception as e:
+        print(f"✗ Transformers model test failed: {str(e)}")
+        return False
+
+def check_langchain_components():
+    """Check critical LangChain components"""
+    print("\n=== LangChain Check ===")
     try:
-        from sentence_transformers import SentenceTransformer
-        print(f"SentenceTransformer version: {SentenceTransformer.__version__}")
-        
-        # Try loading a lightweight default model to see if it works
-        print("Testing with a default model...")
-        model = SentenceTransformer('all-MiniLM-L6-v2')
-        test_embedding = model.encode("Test sentence to check if embeddings work")
-        print(f"✓ Successfully created embedding with shape: {test_embedding.shape}")
+        from langchain_core.messages import HumanMessage
+        from langchain_core.prompts import ChatPromptTemplate
+        from langgraph.graph import StateGraph
+        print("✓ LangChain core components imported successfully")
         return True
     except Exception as e:
-        print(f"✗ SentenceTransformer test failed: {str(e)}")
+        print(f"✗ LangChain check failed: {str(e)}")
         return False
 
 def print_system_info():
@@ -59,37 +76,41 @@ def main():
     """Main function to run checks"""
     print("=== Dependency Check ===")
     print_system_info()
-    
+
     # Get all installed packages
     installed_packages = {pkg.key: pkg.version for pkg in pkg_resources.working_set}
-    
+
     # Check for specific dependencies
     key_packages = [
-        "sentence-transformers",
-        "streamlit", 
-        "langchain-core", 
+        "transformers",
+        "torch",
+        "streamlit",
+        "langchain-core",
         "langchain-openai",
         "langgraph",
         "numpy",
         "scipy"
     ]
-    
+
     print("\n=== Package Versions ===")
     for pkg in key_packages:
         version = installed_packages.get(pkg, "Not installed")
         print(f"{pkg}: {version}")
-    
+
     # Test imports
     imports_ok = check_imports()
     
-    # Test SentenceTransformer
-    st_ok = check_sentence_transformer()
+    # Test transformers model
+    tf_ok = check_transformers_model()
     
+    # Test LangChain
+    lc_ok = check_langchain_components()
+
     # Final result
-    if imports_ok and st_ok:
+    if imports_ok and tf_ok and lc_ok:
         print("\n✓ All critical checks passed")
     else:
         print("\n✗ Some checks failed, see details above")
 
 if __name__ == "__main__":
-    main() 
+    main()

requirements.txt

@@ -1,15 +1,21 @@
+# Core dependencies
+transformers==4.51.3
+torch==2.3.0+cpu
+numpy==1.26.4
+scipy==1.13.0
+streamlit==1.36.0
+
+# LangChain ecosystem
+langchain-core==0.2.4
 langchain-community==0.3.14
 langchain-openai==0.2.14
-langchain-qdrant>=0.2.0
 langgraph==0.2.61
-numpy<2.0.0,>=1.22.4
-unstructured>=0.14.8
-arxiv>=1.4.0
-streamlit>=1.38.0
-tiktoken>=0.6.0
-python-dotenv>=1.0.1
-qdrant-client>=1.7.0
-scipy>=1.10.0
-sentence-transformers==2.3.0 
-transformers>=4.51.3
-torch>=2.0.1
+
+# Hugging Face integration
+huggingface-hub==0.23.2
+
+# Additional utilities
+python-dotenv==1.0.1
+arxiv==1.4.9
+tiktoken==0.7.0
+unstructured==0.14.8