Upload 20 files

backend/.env

@@ -0,0 +1,3 @@
+DATABASE_URL=sqlite:///./rfp.db
+HUGGINGFACE_API_KEY=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
+HUGGINGFACE_MODEL=meta-llama/Meta-Llama-3.1-8B-Instruct

backend/__init__.py

@@ -0,0 +1 @@
+# Backend package

backend/app/__init__.py

@@ -0,0 +1 @@
+# Empty __init__.py files for Python package structure

backend/app/config.py

@@ -0,0 +1,20 @@
+from pydantic_settings import BaseSettings
+from pathlib import Path
+
+# Get the backend directory path
+BACKEND_DIR = Path(__file__).parent.parent
+ENV_FILE = BACKEND_DIR / ".env"
+
+class Settings(BaseSettings):
+    DATABASE_URL: str = "sqlite:///./rfp.db"
+    
+    # HuggingFace API settings
+    HUGGINGFACE_API_KEY: str = ""
+    HUGGINGFACE_MODEL: str = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+    
+    class Config:
+        env_file = str(ENV_FILE)
+        env_file_encoding = 'utf-8'
+        extra = 'ignore'  # Ignore extra fields from .env
+
+settings = Settings()

backend/app/database.py

@@ -0,0 +1,16 @@
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from app.config import settings
+
+engine = create_engine(settings.DATABASE_URL)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/main.py

@@ -0,0 +1,75 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+from app.database import engine, Base
+from app.routers import knowledge_base, rfp_processing, chatbot, health
+from pathlib import Path
+import os
+
+# Create database tables
+Base.metadata.create_all(bind=engine)
+
+app = FastAPI(
+    title="RFP Management System",
+    description="AI-powered RFP management with intelligent answer generation",
+    version="1.0.0"
+)
+
+# CORS middleware for frontend
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:5173", "http://localhost:5174", "http://localhost:3000"],  # SvelteKit default ports
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(health.router)
+app.include_router(knowledge_base.router)
+app.include_router(rfp_processing.router)
+app.include_router(chatbot.router)
+
+# Serve static frontend files
+FRONTEND_BUILD_DIR = Path(__file__).parent.parent.parent / "frontend" / "build"
+
+# Check if we're in deployment (frontend/build exists at different path)
+if not FRONTEND_BUILD_DIR.exists():
+    # In Docker deployment, frontend is at /app/frontend/build
+    FRONTEND_BUILD_DIR = Path("/app/frontend/build")
+
+if FRONTEND_BUILD_DIR.exists():
+    # Mount static files
+    app.mount("/_app", StaticFiles(directory=str(FRONTEND_BUILD_DIR / "_app")), name="app-assets")
+    
+    # Serve index.html for root and all other routes (SPA fallback)
+    @app.get("/")
+    async def serve_frontend():
+        return FileResponse(str(FRONTEND_BUILD_DIR / "index.html"))
+    
+    @app.get("/{full_path:path}")
+    async def serve_spa(full_path: str):
+        # Don't intercept API routes
+        if full_path.startswith("api/"):
+            # Let FastAPI handle API routes normally
+            from fastapi import HTTPException
+            raise HTTPException(status_code=404, detail="Not found")
+        
+        # Serve static files if they exist
+        file_path = FRONTEND_BUILD_DIR / full_path
+        if file_path.is_file():
+            return FileResponse(str(file_path))
+        # Otherwise serve index.html for SPA routing
+        return FileResponse(str(FRONTEND_BUILD_DIR / "index.html"))
+else:
+    # Fallback if frontend not found
+    @app.get("/")
+    def read_root():
+        return {
+            "message": "RFP Management System API",
+            "version": "1.0.0",
+            "docs": "/docs"
+        }
+
+

backend/app/main_hf.py

@@ -0,0 +1,45 @@
+import os
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from app.routers import rfp_processing
+from app.routers import knowledge_base
+from app.database import engine, Base
+
+# Create tables
+Base.metadata.create_all(bind=engine)
+
+app = FastAPI(
+    title="Sedna RFP Management System",
+    description="AI-powered RFP response generation system",
+    version="1.0.0"
+)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(knowledge_base.router)
+app.include_router(rfp_processing.router)
+
+# Serve frontend static files if available
+if os.path.exists("../frontend/build"):
+    app.mount("/", StaticFiles(directory="../frontend/build", html=True), name="frontend")
+
+@app.get("/health")
+def health_check():
+    return {"status": "healthy"}
+
+@app.get("/api/health")
+def api_health_check():
+    return {
+        "status": "healthy",
+        "ai_service": "huggingface",
+        "database": "sqlite"
+    }

backend/app/models.py

@@ -0,0 +1,51 @@
+from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey, Float
+from sqlalchemy.orm import relationship
+from datetime import datetime
+from app.database import Base
+
+class RFP(Base):
+    __tablename__ = "rfps"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    title = Column(String(500), nullable=False)
+    description = Column(Text)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+    
+    questions = relationship("Question", back_populates="rfp", cascade="all, delete-orphan")
+
+class Question(Base):
+    __tablename__ = "questions"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    rfp_id = Column(Integer, ForeignKey("rfps.id"), nullable=False)
+    section = Column(String(200))
+    question_id = Column(String(50))  # Custom question ID like "Q1.1"
+    question_text = Column(Text, nullable=False)
+    answer_text = Column(Text, nullable=False)
+    page_start = Column(Integer)
+    page_end = Column(Integer)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+    
+    rfp = relationship("RFP", back_populates="questions")
+    embedding = relationship("QuestionEmbedding", back_populates="question", uselist=False, cascade="all, delete-orphan")
+
+class QuestionEmbedding(Base):
+    __tablename__ = "question_embeddings"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    question_id = Column(Integer, ForeignKey("questions.id"), nullable=False, unique=True)
+    embedding = Column(Text, nullable=False)  # JSON serialized numpy array
+    
+    question = relationship("Question", back_populates="embedding")
+
+class UploadedRFP(Base):
+    __tablename__ = "uploaded_rfps"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    filename = Column(String(500), nullable=False)
+    rfp_name = Column(String(500))  # Friendly name for the RFP
+    file_path = Column(String(1000), nullable=False)
+    uploaded_at = Column(DateTime, default=datetime.utcnow)
+    text_content = Column(Text)  # Extracted text from first 3 pages or full document

backend/app/routers/__init__.py

@@ -0,0 +1 @@
+# Empty __init__.py files for Python package structure

backend/app/routers/chatbot.py

@@ -0,0 +1,97 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import List
+from app.database import get_db
+from app.services.huggingface_service import huggingface_service
+from app.models import UploadedRFP
+import pdfplumber
+
+router = APIRouter(prefix="/api/chatbot", tags=["Chatbot"])
+
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+
+class ChatRequest(BaseModel):
+    message: str
+    conversation_history: List[ChatMessage] = []
+
+class ChatResponse(BaseModel):
+    response: str
+
+@router.post("/chat", response_model=ChatResponse)
+async def chat_with_assistant(request: ChatRequest):
+    """
+    AI chatbot that helps understand uploaded RFPs and answer questions about requirements.
+    Uses HuggingFace Inference API with the uploaded RFP content.
+    """
+    try:
+        db = next(get_db())
+        
+        # Get the most recent uploaded RFP
+        uploaded_rfp = db.query(UploadedRFP).order_by(UploadedRFP.id.desc()).first()
+        
+        if not uploaded_rfp:
+            return ChatResponse(
+                response="I don't have access to any uploaded RFPs yet. Please upload an RFP document first so I can help you understand its requirements."
+            )
+        
+        # Extract RFP content for context
+        rfp_context = ""
+        if uploaded_rfp.file_path:
+            try:
+                with pdfplumber.open(uploaded_rfp.file_path) as pdf:
+                    # Extract first 5 pages for context
+                    pages_text = []
+                    for page in pdf.pages[:5]:
+                        text = page.extract_text()
+                        if text:
+                            pages_text.append(text)
+                    rfp_context = "\n\n".join(pages_text)
+            except Exception as e:
+                print(f"Error extracting RFP content: {e}")
+                rfp_context = "Unable to extract RFP content"
+        
+        # Create comprehensive prompt for the AI assistant
+        system_context = f"""You are an expert RFP assistant for Sedna Consulting Group helping the team understand the uploaded RFP: "{uploaded_rfp.rfp_name}".
+
+Answer questions about:
+- RFP requirements and specifications
+- Deadlines and submission guidelines  
+- Technical requirements
+- What information Sedna needs to provide
+
+Be conversational, professional, and cite specific sections when possible.
+
+RFP CONTENT:
+{rfp_context[:3000]}"""
+
+        # Use HuggingFace Inference API
+        print("Using HuggingFace API for chat...")
+        
+        # Build conversation history
+        history = [
+            {'role': msg.role, 'content': msg.content}
+            for msg in request.conversation_history[-6:]
+        ]
+        
+        response_text = huggingface_service.generate_chat_response(
+            system_prompt=system_context,
+            user_message=request.message,
+            conversation_history=history,
+            max_tokens=800,
+            temperature=0.7
+        )
+
+        # If HF returned a diagnostic error string, surface it as a 502 so frontend can show it
+        if isinstance(response_text, str) and response_text.startswith("Error:"):
+            # Provide actionable details but avoid leaking secrets
+            raise HTTPException(status_code=502, detail=response_text)
+
+        return ChatResponse(response=response_text)
+        
+    except Exception as e:
+        print(f"Chatbot error: {e}")
+        return ChatResponse(
+            response="I apologize, but I encountered an error processing your question. Please try again or be more specific about what section of the RFP you're interested in."
+        )

backend/app/routers/health.py

@@ -0,0 +1,98 @@
+from fastapi import APIRouter
+from pydantic import BaseModel
+from app.services.huggingface_service import huggingface_service
+from app.config import settings
+
+router = APIRouter(prefix="/api/health", tags=["Health"])
+
+class HealthStatus(BaseModel):
+    status: str
+    message: str
+    details: dict = {}
+
+@router.get("/", response_model=HealthStatus)
+async def health_check():
+    """Basic health check for the API"""
+    return HealthStatus(
+        status="healthy",
+        message="API is running",
+        details={"version": "1.0.0"}
+    )
+
+@router.get("/ai", response_model=HealthStatus)
+async def ai_health_check():
+    """
+    Check if HuggingFace AI model is accessible and working
+    Returns detailed diagnostics about model availability, token validity, etc.
+    """
+    # Check if service is configured
+    if not huggingface_service.is_configured():
+        return HealthStatus(
+            status="unhealthy",
+            message="HuggingFace API key not configured",
+            details={
+                "model": settings.HUGGINGFACE_MODEL,
+                "api_key_set": bool(settings.HUGGINGFACE_API_KEY),
+                "recommendation": "Set HUGGINGFACE_API_KEY in Space secrets"
+            }
+        )
+    
+    # Check model availability using the service's built-in check
+    diag = huggingface_service._check_model_available()
+    
+    if diag is None:
+        # Model is accessible
+        return HealthStatus(
+            status="healthy",
+            message="AI model is accessible and ready",
+            details={
+                "model": settings.HUGGINGFACE_MODEL.strip(),
+                "api_status": "connected"
+            }
+        )
+    else:
+        # Model check failed
+        error_msg = str(diag).lower()
+        
+        if "401" in error_msg or "unauthorized" in error_msg:
+            return HealthStatus(
+                status="unhealthy",
+                message="API key unauthorized",
+                details={
+                    "model": settings.HUGGINGFACE_MODEL.strip(),
+                    "error": diag,
+                    "recommendation": "Check your HUGGINGFACE_API_KEY token has appropriate permissions"
+                }
+            )
+        
+        elif "403" in error_msg or "forbidden" in error_msg:
+            return HealthStatus(
+                status="unhealthy",
+                message="Access forbidden - model may be gated",
+                details={
+                    "model": settings.HUGGINGFACE_MODEL.strip(),
+                    "error": diag,
+                    "recommendation": f"Ensure you've accepted the model license at https://huggingface.co/{settings.HUGGINGFACE_MODEL.strip()}"
+                }
+            )
+        
+        elif "503" in error_msg or "loading" in error_msg:
+            return HealthStatus(
+                status="degraded",
+                message="Model is loading",
+                details={
+                    "model": settings.HUGGINGFACE_MODEL.strip(),
+                    "error": diag,
+                    "recommendation": "Wait a moment and try again. Model is initializing on HuggingFace servers."
+                }
+            )
+        
+        else:
+            return HealthStatus(
+                status="unhealthy",
+                message="Error connecting to HuggingFace API",
+                details={
+                    "model": settings.HUGGINGFACE_MODEL.strip(),
+                    "error": diag
+                }
+            )

backend/app/routers/knowledge_base.py

@@ -0,0 +1,171 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.orm import Session
+from typing import List
+from app.database import get_db
+from app.models import RFP, Question
+from app.schemas import (
+    RFPCreate, RFPUpdate, RFPResponse,
+    QuestionCreate, QuestionUpdate, QuestionResponse, QuestionWithRFP
+)
+from app.services.similarity_service import similarity_service
+
+router = APIRouter(prefix="/api/knowledge-base", tags=["Knowledge Base"])
+
+# RFP Endpoints
+@router.post("/rfps", response_model=RFPResponse)
+def create_rfp(rfp: RFPCreate, db: Session = Depends(get_db)):
+    """Create a new RFP"""
+    db_rfp = RFP(**rfp.model_dump())
+    db.add(db_rfp)
+    db.commit()
+    db.refresh(db_rfp)
+    return db_rfp
+
+@router.get("/rfps", response_model=List[RFPResponse])
+def list_rfps(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
+    """List all RFPs"""
+    rfps = db.query(RFP).offset(skip).limit(limit).all()
+    return rfps
+
+@router.get("/rfps/{rfp_id}", response_model=RFPResponse)
+def get_rfp(rfp_id: int, db: Session = Depends(get_db)):
+    """Get a specific RFP"""
+    rfp = db.query(RFP).filter(RFP.id == rfp_id).first()
+    if not rfp:
+        raise HTTPException(status_code=404, detail="RFP not found")
+    return rfp
+
+@router.put("/rfps/{rfp_id}", response_model=RFPResponse)
+def update_rfp(rfp_id: int, rfp_update: RFPUpdate, db: Session = Depends(get_db)):
+    """Update an RFP"""
+    db_rfp = db.query(RFP).filter(RFP.id == rfp_id).first()
+    if not db_rfp:
+        raise HTTPException(status_code=404, detail="RFP not found")
+    
+    for key, value in rfp_update.model_dump(exclude_unset=True).items():
+        setattr(db_rfp, key, value)
+    
+    db.commit()
+    db.refresh(db_rfp)
+    return db_rfp
+
+@router.delete("/rfps/{rfp_id}")
+def delete_rfp(rfp_id: int, db: Session = Depends(get_db)):
+    """Delete an RFP and all its questions"""
+    db_rfp = db.query(RFP).filter(RFP.id == rfp_id).first()
+    if not db_rfp:
+        raise HTTPException(status_code=404, detail="RFP not found")
+    
+    db.delete(db_rfp)
+    db.commit()
+    return {"message": "RFP deleted successfully"}
+
+# Question Endpoints
+@router.post("/questions", response_model=QuestionResponse)
+def create_question(question: QuestionCreate, db: Session = Depends(get_db)):
+    """Create a new question and generate its embedding"""
+    # Verify RFP exists
+    rfp = db.query(RFP).filter(RFP.id == question.rfp_id).first()
+    if not rfp:
+        raise HTTPException(status_code=404, detail="RFP not found")
+    
+    # Create question
+    db_question = Question(**question.model_dump())
+    db.add(db_question)
+    db.commit()
+    db.refresh(db_question)
+    
+    # Generate and save embedding immediately
+    try:
+        print(f"Generating embedding for question {db_question.id}...")
+        similarity_service.save_question_embedding(db, db_question.id, db_question.question_text)
+        print(f"Embedding generated successfully for question {db_question.id}")
+    except Exception as e:
+        print(f"ERROR: Failed to generate embedding for question {db_question.id}: {e}")
+        # Don't fail the request, but log the error
+    
+    return db_question
+
+@router.get("/questions", response_model=List[QuestionWithRFP])
+def list_questions(
+    skip: int = 0, 
+    limit: int = 100, 
+    rfp_id: int = None,
+    db: Session = Depends(get_db)
+):
+    """List all questions, optionally filtered by RFP"""
+    query = db.query(Question)
+    
+    if rfp_id:
+        query = query.filter(Question.rfp_id == rfp_id)
+    
+    questions = query.offset(skip).limit(limit).all()
+    return questions
+
+@router.get("/questions/{question_id}", response_model=QuestionWithRFP)
+def get_question(question_id: int, db: Session = Depends(get_db)):
+    """Get a specific question"""
+    question = db.query(Question).filter(Question.id == question_id).first()
+    if not question:
+        raise HTTPException(status_code=404, detail="Question not found")
+    return question
+
+@router.put("/questions/{question_id}", response_model=QuestionResponse)
+def update_question(
+    question_id: int, 
+    question_update: QuestionUpdate, 
+    db: Session = Depends(get_db)
+):
+    """Update a question and regenerate its embedding if text changed"""
+    db_question = db.query(Question).filter(Question.id == question_id).first()
+    if not db_question:
+        raise HTTPException(status_code=404, detail="Question not found")
+    
+    update_data = question_update.model_dump(exclude_unset=True)
+    question_text_changed = "question_text" in update_data
+    
+    for key, value in update_data.items():
+        setattr(db_question, key, value)
+    
+    db.commit()
+    db.refresh(db_question)
+    
+    # Regenerate embedding if question text changed
+    if question_text_changed:
+        similarity_service.save_question_embedding(db, db_question.id, db_question.question_text)
+    
+    return db_question
+
+@router.delete("/questions/{question_id}")
+def delete_question(question_id: int, db: Session = Depends(get_db)):
+    """Delete a question"""
+    question = db.query(Question).filter(Question.id == question_id).first()
+    if not question:
+        raise HTTPException(status_code=404, detail="Question not found")
+    
+    db.delete(question)
+    db.commit()
+    return {"message": "Question deleted successfully"}
+
+@router.post("/generate-embeddings")
+def generate_all_embeddings(db: Session = Depends(get_db)):
+    """Generate embeddings for all questions that don't have them"""
+    questions = db.query(Question).all()
+    generated = 0
+    failed = 0
+    
+    for question in questions:
+        try:
+            similarity_service.save_question_embedding(db, question.id, question.question_text)
+            generated += 1
+            print(f"Generated embedding for question {question.id}")
+        except Exception as e:
+            failed += 1
+            print(f"Failed to generate embedding for question {question.id}: {e}")
+    
+    return {
+        "message": "Embedding generation complete",
+        "total_questions": len(questions),
+        "generated": generated,
+        "failed": failed
+    }

backend/app/routers/rfp_processing.py

@@ -0,0 +1,185 @@
+from fastapi import APIRouter, Depends, HTTPException, UploadFile, File
+from sqlalchemy.orm import Session
+from typing import List
+from app.database import get_db
+from app.models import UploadedRFP, Question
+from app.schemas import ProcessQuestionRequest, AIAnswerResponse, SimilarQuestion
+from app.services.pdf_service import pdf_service
+from app.services.similarity_service import similarity_service
+from app.services.huggingface_service import huggingface_service
+
+router = APIRouter(prefix="/api/rfp-processing", tags=["RFP Processing"])
+
+@router.post("/upload")
+async def upload_rfp(file: UploadFile = File(...), db: Session = Depends(get_db)):
+    """Upload a new RFP PDF and extract text from first 3 pages"""
+    if not file.filename.endswith('.pdf'):
+        raise HTTPException(status_code=400, detail="Only PDF files are allowed")
+    
+    uploaded_rfp = await pdf_service.save_and_extract_pdf(file, db, extract_pages=3)
+    
+    return {
+        "id": uploaded_rfp.id,
+        "filename": uploaded_rfp.filename,
+        "uploaded_at": uploaded_rfp.uploaded_at,
+        "text_preview": uploaded_rfp.text_content[:500] + "..." if len(uploaded_rfp.text_content) > 500 else uploaded_rfp.text_content
+    }
+
+@router.get("/uploaded-rfps")
+def list_uploaded_rfps(skip: int = 0, limit: int = 50, db: Session = Depends(get_db)):
+    """List all uploaded RFPs"""
+    rfps = db.query(UploadedRFP).offset(skip).limit(limit).all()
+    return [
+        {
+            "id": rfp.id,
+            "filename": rfp.filename,
+            "rfp_name": rfp.rfp_name if hasattr(rfp, 'rfp_name') and rfp.rfp_name else rfp.filename.rsplit('.', 1)[0],
+            "uploaded_at": rfp.uploaded_at
+        }
+        for rfp in rfps
+    ]
+
+@router.delete("/uploaded-rfps/{rfp_id}")
+def delete_uploaded_rfp(rfp_id: int, db: Session = Depends(get_db)):
+    """Delete an uploaded RFP"""
+    uploaded_rfp = db.query(UploadedRFP).filter(UploadedRFP.id == rfp_id).first()
+    if not uploaded_rfp:
+        raise HTTPException(status_code=404, detail="Uploaded RFP not found")
+    
+    # Delete the file from disk if it exists
+    import os
+    if uploaded_rfp.file_path and os.path.exists(uploaded_rfp.file_path):
+        try:
+            os.remove(uploaded_rfp.file_path)
+        except Exception as e:
+            print(f"Error deleting file: {e}")
+    
+    db.delete(uploaded_rfp)
+    db.commit()
+    
+    return {"message": "Uploaded RFP deleted successfully"}
+
+@router.post("/process-question", response_model=AIAnswerResponse)
+async def process_question(request: ProcessQuestionRequest, db: Session = Depends(get_db)):
+    """
+    Process a question against a new RFP:
+    1. Find similar questions from database
+    2. Get new RFP content (if provided)
+    3. Generate AI answer combining both
+    4. Provide gap analysis
+    """
+    # Get uploaded RFP (optional)
+    uploaded_rfp = None
+    new_rfp_content = ""
+    
+    if request.uploaded_rfp_id:
+        uploaded_rfp = db.query(UploadedRFP).filter(
+            UploadedRFP.id == request.uploaded_rfp_id
+        ).first()
+        
+        if not uploaded_rfp:
+            raise HTTPException(status_code=404, detail="Uploaded RFP not found")
+        
+        new_rfp_content = uploaded_rfp.text_content
+    
+    # Check for exact match first
+    exact_match = similarity_service.check_exact_match(db, request.question_text)
+    
+    if exact_match:
+        # Return exact match answer
+        return AIAnswerResponse(
+            ai_answer=exact_match.answer_text,
+            gap_analysis="Exact match found in database. This is the stored answer from previous RFP." if not new_rfp_content else "Exact match found. No new information in current RFP.",
+            similar_questions=[
+                SimilarQuestion(
+                    question_id=exact_match.id,
+                    question_text=exact_match.question_text,
+                    answer_text=exact_match.answer_text,
+                    page_start=exact_match.page_start,
+                    page_end=exact_match.page_end,
+                    rfp_title=exact_match.rfp.title,
+                    similarity_score=1.0
+                )
+            ],
+            new_rfp_content=new_rfp_content[:1000] + "..." if new_rfp_content else "No new RFP uploaded"
+        )
+    
+    # Find similar questions - get more matches for AI to consider
+    similar_questions = similarity_service.find_similar_questions(
+        db, 
+        request.question_text,
+        top_k=10,  # Get more answers for AI to reference
+        threshold=0.15  # Lower threshold to include more potential matches
+    )
+    
+    # Format similar questions for response
+    similar_questions_list = [
+        SimilarQuestion(
+            question_id=q.id,
+            question_text=q.question_text,
+            answer_text=q.answer_text,
+            page_start=q.page_start,
+            page_end=q.page_end,
+            rfp_title=q.rfp.title,
+            similarity_score=score
+        )
+        for q, score in similar_questions
+    ]
+    
+    print(f"Found {len(similar_questions_list)} similar questions")
+    for sq in similar_questions_list:
+        print(f"  - {sq.question_text} (score: {sq.similarity_score:.2f})")
+    
+    # Prepare previous answers for AI
+    previous_answers = [
+        {
+            "question_text": q.question_text,
+            "answer_text": q.answer_text,
+            "page_start": q.page_start,
+            "page_end": q.page_end,
+            "rfp_title": q.rfp.title,
+            "similarity_score": score
+        }
+        for q, score in similar_questions
+    ]
+    
+    # Generate AI answer with HuggingFace
+    prompt = f"""Question: {request.question_text}
+
+New RFP Content:
+{new_rfp_content if new_rfp_content else "No new RFP content provided"}
+
+Previous Similar Answers:
+{chr(10).join([f"- {ans['answer_text']} (from {ans['rfp_title']}, score: {ans['similarity_score']:.2f})" for ans in previous_answers[:5]])}
+
+{request.additional_context if request.additional_context else ""}
+
+Provide a comprehensive answer and gap analysis."""
+
+    ai_answer = huggingface_service.generate_chat_response(prompt)
+    
+    ai_response = {
+        'ai_answer': ai_answer,
+        'gap_analysis': "Generated using HuggingFace Mistral-7B model based on available context."
+    }
+    
+    return AIAnswerResponse(
+        ai_answer=ai_response['ai_answer'],
+        gap_analysis=ai_response['gap_analysis'],
+        similar_questions=similar_questions_list,
+        new_rfp_content=new_rfp_content[:1000] + "..." if new_rfp_content else "No new RFP uploaded"
+    )
+
+@router.get("/uploaded-rfps/{rfp_id}/content")
+def get_rfp_content(rfp_id: int, db: Session = Depends(get_db)):
+    """Get the full extracted content of an uploaded RFP"""
+    uploaded_rfp = db.query(UploadedRFP).filter(UploadedRFP.id == rfp_id).first()
+    
+    if not uploaded_rfp:
+        raise HTTPException(status_code=404, detail="Uploaded RFP not found")
+    
+    return {
+        "id": uploaded_rfp.id,
+        "filename": uploaded_rfp.filename,
+        "content": uploaded_rfp.text_content
+    }

backend/app/schemas.py

@@ -0,0 +1,73 @@
+from pydantic import BaseModel
+from typing import Optional
+from datetime import datetime
+
+# RFP Schemas
+class RFPBase(BaseModel):
+    title: str
+    description: Optional[str] = None
+
+class RFPCreate(RFPBase):
+    pass
+
+class RFPUpdate(RFPBase):
+    pass
+
+class RFPResponse(RFPBase):
+    id: int
+    created_at: datetime
+    updated_at: datetime
+    
+    class Config:
+        from_attributes = True
+
+# Question Schemas
+class QuestionBase(BaseModel):
+    section: Optional[str] = None
+    question_id: Optional[str] = None
+    question_text: str
+    answer_text: str
+    page_start: Optional[int] = None
+    page_end: Optional[int] = None
+
+class QuestionCreate(QuestionBase):
+    rfp_id: int
+
+class QuestionUpdate(QuestionBase):
+    pass
+
+class QuestionResponse(QuestionBase):
+    id: int
+    rfp_id: int
+    created_at: datetime
+    updated_at: datetime
+    
+    class Config:
+        from_attributes = True
+
+class QuestionWithRFP(QuestionResponse):
+    rfp: RFPResponse
+    
+    class Config:
+        from_attributes = True
+
+# New RFP Processing Schemas
+class SimilarQuestion(BaseModel):
+    question_id: int
+    question_text: str
+    answer_text: str
+    page_start: Optional[int]
+    page_end: Optional[int]
+    rfp_title: str
+    similarity_score: float
+
+class AIAnswerResponse(BaseModel):
+    ai_answer: str
+    gap_analysis: str
+    similar_questions: list[SimilarQuestion]
+    new_rfp_content: str
+
+class ProcessQuestionRequest(BaseModel):
+    uploaded_rfp_id: Optional[int] = None
+    question_text: str
+    additional_context: Optional[str] = None

backend/app/services/__init__.py

@@ -0,0 +1 @@
+# Empty __init__.py files for Python package structure

backend/app/services/hf_ollama_service.py

@@ -0,0 +1,125 @@
+import os
+import requests
+from typing import List, Dict
+
+class HuggingFaceOllamaService:
+    """
+    Modified Ollama service to use Hugging Face Inference API instead of local Ollama
+    """
+    def __init__(self):
+        self.api_token = os.getenv("HF_TOKEN", "")
+        self.model = "meta-llama/Llama-3.1-8B-Instruct"
+        self.api_url = f"https://api-inference.huggingface.co/models/{self.model}"
+    
+    def generate_answer(
+        self, 
+        question: str, 
+        new_rfp_content: str, 
+        previous_answers: List[dict],
+        additional_context: str = None
+    ) -> Dict[str, str]:
+        """
+        Generate an AI answer using Hugging Face Inference API
+        """
+        # Build context from previous answers
+        previous_context = self._format_previous_answers(previous_answers)
+        
+        # Build additional context section
+        additional_section = ""
+        if additional_context:
+            additional_section = f"""
+
+ADDITIONAL REQUIREMENTS:
+{additional_context}
+
+You MUST include the above additional requirements in your answer.
+"""
+        
+        # Create prompt
+        prompt = f"""You are Sedna's RFP response writer. Write a professional, polished answer to the RFP question as if this is the FINAL response that will be submitted to the client.
+
+QUESTION FROM NEW RFP:
+{question}
+
+NEW RFP CONTENT (uploaded document - context only):
+{new_rfp_content[:2000] if new_rfp_content and new_rfp_content.strip() != "No new RFP content provided" else "No new RFP document uploaded"}
+
+SEDNA'S PREVIOUS ANSWERS FROM KNOWLEDGE BASE:
+{previous_context[:3000]}
+{additional_section}
+
+YOUR TASK:
+Write a clean, professional RFP response for Sedna. Use specific facts from the knowledge base but present them naturally as part of the answer. Focus on what the client is asking for.
+
+Write 3-5 comprehensive paragraphs addressing all parts of the question. If information is missing, simply note what additional details you'd be happy to discuss.
+
+ANSWER:"""
+
+        # Call Hugging Face API
+        headers = {"Authorization": f"Bearer {self.api_token}"}
+        payload = {
+            "inputs": prompt,
+            "parameters": {
+                "max_new_tokens": 1000,
+                "temperature": 0.3,
+                "top_p": 0.9,
+                "return_full_text": False
+            }
+        }
+        
+        try:
+            response = requests.post(self.api_url, headers=headers, json=payload, timeout=60)
+            response.raise_for_status()
+            result = response.json()
+            
+            if isinstance(result, list) and len(result) > 0:
+                full_response = result[0].get("generated_text", "")
+            else:
+                full_response = "Unable to generate response. Please try again."
+            
+            # Parse response
+            answer, gap_analysis = self._parse_response(full_response)
+            
+            return {
+                'ai_answer': answer,
+                'gap_analysis': gap_analysis
+            }
+        except Exception as e:
+            print(f"Error calling Hugging Face API: {e}")
+            return {
+                'ai_answer': f"Error generating AI response: {str(e)}",
+                'gap_analysis': "Unable to analyze gaps due to API error"
+            }
+    
+    def _format_previous_answers(self, previous_answers: List[dict]) -> str:
+        """Format previous answers for context"""
+        if not previous_answers:
+            return "No previous similar answers found."
+        
+        formatted = []
+        for idx, ans in enumerate(previous_answers[:5], 1):  # Limit to 5 for token efficiency
+            formatted.append(f"""
+Previous Answer {idx}:
+Question: {ans.get('question_text', '')}
+Answer: {ans.get('answer_text', '')[:500]}...
+""")
+        
+        return "\n".join(formatted)
+    
+    def _parse_response(self, response: str) -> tuple[str, str]:
+        """Parse the AI response to extract answer and gap analysis"""
+        answer = ""
+        gap_analysis = ""
+        
+        if "GAP ANALYSIS:" in response:
+            parts = response.split("GAP ANALYSIS:")
+            answer = parts[0].strip()
+            gap_analysis = parts[1].strip() if len(parts) > 1 else ""
+        else:
+            answer = response.strip()
+            gap_analysis = "No gap analysis provided"
+        
+        return answer, gap_analysis
+
+# Create singleton instance
+hf_ollama_service = HuggingFaceOllamaService()

backend/app/services/huggingface_service.py

@@ -0,0 +1,128 @@
+import sys
+from typing import List, Optional
+
+import requests
+
+from app.config import settings
+
+
+class HuggingFaceService:
+    def __init__(self):
+        self.api_key = settings.HUGGINGFACE_API_KEY
+        # Make sure this is something like "meta-llama/Llama-3.1-8B-Instruct"
+        self.model = settings.HUGGINGFACE_MODEL.strip()
+        # Use serverless inference endpoint directly
+        self.api_url = f"https://api-inference.huggingface.co/models/{self.model}"
+        
+        # Common headers for requests
+        self.headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def _check_model_available(self) -> Optional[str]:
+        """Check if the model is available."""
+        try:
+            resp = requests.post(
+                self.api_url,
+                headers=self.headers,
+                json={"inputs": "test", "parameters": {"max_new_tokens": 1}},
+                timeout=15,
+            )
+            resp.raise_for_status()
+            return None
+        except Exception as e:
+            return f"Model check failed: {str(e)}"
+
+    def generate_chat_response(
+        self,
+        system_prompt: str,
+        user_message: str,
+        conversation_history: List[dict] = None,
+        max_tokens: int = 1000,
+        temperature: float = 0.7,
+    ) -> str:
+        """Generate a chat response using HuggingFace Inference API."""
+
+        # Build a simple prompt
+        prompt_parts = []
+        
+        if system_prompt:
+            prompt_parts.append(f"<|system|>\n{system_prompt}\n")
+        
+        if conversation_history:
+            for msg in conversation_history[-6:]:
+                role = msg.get("role", "user")
+                content = msg.get("content", "")
+                if role == "user":
+                    prompt_parts.append(f"<|user|>\n{content}\n")
+                else:
+                    prompt_parts.append(f"<|assistant|>\n{content}\n")
+        
+        prompt_parts.append(f"<|user|>\n{user_message}\n<|assistant|>\n")
+        prompt = "".join(prompt_parts)
+
+        payload = {
+            "inputs": prompt,
+            "parameters": {
+                "max_new_tokens": max_tokens,
+                "temperature": temperature,
+                "return_full_text": False,
+            }
+        }
+
+        try:
+            resp = requests.post(
+                self.api_url,
+                headers=self.headers,
+                json=payload,
+                timeout=60,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+
+            # Response format: [{"generated_text": "..."}]
+            if isinstance(data, list) and len(data) > 0:
+                content = data[0].get("generated_text", "")
+            else:
+                content = str(data)
+
+            return content.strip() if content else "AI response was empty."
+
+        except Exception as e:
+            error_msg = str(e)
+            print(f"HuggingFace service error: {error_msg}", file=sys.stderr)
+
+            lower = error_msg.lower()
+            if "401" in lower or "unauthorized" in lower:
+                return (
+                    "API key error: unauthorized. "
+                    "Check your HUGGINGFACE_API_KEY in Space secrets."
+                )
+            elif "403" in lower or "forbidden" in lower:
+                return (
+                    f"Access forbidden. Ensure you've accepted the model "
+                    f"license at https://huggingface.co/{self.model}"
+                )
+            elif "402" in lower or "payment required" in lower:
+                return (
+                    "Payment required for this model/provider. "
+                    "Check your Hugging Face Inference / credits."
+                )
+            elif "503" in lower or "loading" in lower or "temporarily unavailable" in lower:
+                return (
+                    "The AI model or provider is temporarily unavailable or loading. "
+                    "Try again in a minute."
+                )
+            else:
+                return f"Error connecting to HuggingFace API: {error_msg}"
+
+    def is_configured(self) -> bool:
+        """Check if HuggingFace is properly configured."""
+        return bool(
+            self.api_key
+            and self.api_key != "your_huggingface_api_key_here"
+            and self.model
+        )
+
+huggingface_service = HuggingFaceService()

backend/app/services/pdf_service.py

@@ -0,0 +1,82 @@
+import pdfplumber
+import os
+from typing import Optional
+from fastapi import UploadFile
+from app.models import UploadedRFP
+from sqlalchemy.orm import Session
+
+class PDFService:
+    def __init__(self, upload_dir: str = "uploads"):
+        self.upload_dir = upload_dir
+        os.makedirs(upload_dir, exist_ok=True)
+    
+    async def save_and_extract_pdf(
+        self, 
+        file: UploadFile, 
+        db: Session,
+        extract_pages: Optional[int] = 3
+    ) -> UploadedRFP:
+        """
+        Save uploaded PDF and extract text from first N pages
+        """
+        # Save file
+        file_path = os.path.join(self.upload_dir, file.filename)
+        
+        with open(file_path, "wb") as buffer:
+            content = await file.read()
+            buffer.write(content)
+        
+        # Extract text
+        extracted_text = self.extract_text_from_pdf(file_path, max_pages=extract_pages)
+        
+        # Generate friendly RFP name from filename (remove .pdf extension)
+        rfp_name = file.filename.rsplit('.', 1)[0] if '.' in file.filename else file.filename
+        
+        # Save to database
+        uploaded_rfp = UploadedRFP(
+            filename=file.filename,
+            rfp_name=rfp_name,
+            file_path=file_path,
+            text_content=extracted_text
+        )
+        db.add(uploaded_rfp)
+        db.commit()
+        db.refresh(uploaded_rfp)
+        
+        return uploaded_rfp
+    
+    def extract_text_from_pdf(self, file_path: str, max_pages: Optional[int] = None) -> str:
+        """
+        Extract text from PDF using pdfplumber
+        """
+        text_content = []
+        
+        with pdfplumber.open(file_path) as pdf:
+            pages_to_process = pdf.pages[:max_pages] if max_pages else pdf.pages
+            
+            for page in pages_to_process:
+                text = page.extract_text()
+                if text:
+                    text_content.append(text)
+        
+        return "\n\n".join(text_content)
+    
+    def extract_specific_pages(self, file_path: str, start_page: int, end_page: int) -> str:
+        """
+        Extract text from specific page range (1-indexed)
+        """
+        text_content = []
+        
+        with pdfplumber.open(file_path) as pdf:
+            # Convert to 0-indexed
+            start_idx = start_page - 1
+            end_idx = end_page
+            
+            for page in pdf.pages[start_idx:end_idx]:
+                text = page.extract_text()
+                if text:
+                    text_content.append(text)
+        
+        return "\n\n".join(text_content)
+
+pdf_service = PDFService()

backend/app/services/similarity_service.py

@@ -0,0 +1,305 @@
+import numpy as np
+import json
+from typing import List, Tuple
+from sqlalchemy.orm import Session
+from app.models import Question, QuestionEmbedding
+
+class SimilaritySearchService:
+    def __init__(self):
+        # HuggingFace model for embeddings
+        self.model = None
+        self._model_loaded = False
+    
+    def _load_model(self):
+        """Load the HuggingFace sentence transformer model"""
+        if not self._model_loaded:
+            try:
+                from sentence_transformers import SentenceTransformer
+                print("Loading HuggingFace sentence-transformers model (all-MiniLM-L6-v2)...")
+                # Using a lightweight, fast model from HuggingFace
+                self.model = SentenceTransformer('all-MiniLM-L6-v2')
+                self._model_loaded = True
+                print("✓ HuggingFace model loaded successfully!")
+                print(f"  Model: all-MiniLM-L6-v2")
+                print(f"  Embedding dimension: 384")
+            except Exception as e:
+                print(f"ERROR: Could not load HuggingFace sentence-transformers: {e}")
+                print("Falling back to dummy embeddings")
+                self._model_loaded = True
+    
+    def generate_embedding(self, text: str) -> np.ndarray:
+        """Generate embedding using HuggingFace model"""
+        self._load_model()
+        if not self.model:
+            print("WARNING: Using random embeddings (model not loaded)")
+            return np.random.rand(384)
+        
+        # Generate embedding using HuggingFace
+        embedding = self.model.encode(text, convert_to_numpy=True)
+        return embedding
+    
+    def save_question_embedding(self, db: Session, question_id: int, question_text: str):
+        """Generate and save embedding for a question"""
+        embedding = self.generate_embedding(question_text)
+        embedding_json = json.dumps(embedding.tolist())
+        
+        # Check if embedding exists
+        existing = db.query(QuestionEmbedding).filter(
+            QuestionEmbedding.question_id == question_id
+        ).first()
+        
+        if existing:
+            existing.embedding = embedding_json
+        else:
+            db_embedding = QuestionEmbedding(
+                question_id=question_id,
+                embedding=embedding_json
+            )
+            db.add(db_embedding)
+        
+        db.commit()
+    
+    def find_similar_questions(
+        self, 
+        db: Session, 
+        question_text: str, 
+        top_k: int = 5,
+        threshold: float = 0.5
+    ) -> List[Tuple[Question, float]]:
+        """
+        Find similar questions using cosine similarity with text-based fallback
+        Returns list of (Question, similarity_score) tuples
+        """
+        similarities = []
+        
+        # FIRST: Check text similarity for all questions, especially short ones
+        # This ensures our special case matching works (Executive Summary -> Vendor Org = 98%)
+        all_questions = db.query(Question).all()
+        text_similarities = []
+        
+        for question in all_questions:
+            text_score = self._text_similarity(question_text, question.question_text)
+            if text_score >= 0.90:  # Very high text match (95%+)
+                print(f"HIGH TEXT MATCH FOUND: '{question.question_text}' = {text_score:.2f}")
+                text_similarities.append((question, text_score))
+        
+        # If we found high-confidence text matches, use those
+        if text_similarities:
+            text_similarities.sort(key=lambda x: x[1], reverse=True)
+            print(f"Using {len(text_similarities)} high-confidence text matches instead of embeddings")
+            return text_similarities[:top_k]
+        
+        try:
+            # Try embedding-based search
+            query_embedding = self.generate_embedding(question_text)
+            all_embeddings = db.query(QuestionEmbedding).all()
+            
+            if all_embeddings and len(all_embeddings) > 0:
+                for emb in all_embeddings:
+                    try:
+                        stored_embedding = np.array(json.loads(emb.embedding))
+                        similarity = self._cosine_similarity(query_embedding, stored_embedding)
+                        
+                        if similarity >= threshold:
+                            similarities.append((emb.question, similarity))
+                    except Exception as e:
+                        print(f"Error processing embedding for question {emb.question_id}: {e}")
+                        continue
+            
+            # Sort by similarity score (descending)
+            similarities.sort(key=lambda x: x[1], reverse=True)
+            
+            if len(similarities) > 0:
+                print(f"Found {len(similarities)} matches using embeddings")
+                return similarities[:top_k]
+        
+        except Exception as e:
+            print(f"Embedding search failed: {e}")
+        
+        # Fallback to text-based search if embeddings failed or no results
+        print("Using text-based fallback search...")
+        all_questions = db.query(Question).all()
+        
+        for question in all_questions:
+            text_similarity = self._text_similarity(question_text, question.question_text)
+            if text_similarity >= threshold:
+                similarities.append((question, text_similarity))
+        
+        # Sort by similarity score (descending)
+        similarities.sort(key=lambda x: x[1], reverse=True)
+        
+        print(f"Found {len(similarities)} matches using text search")
+        return similarities[:top_k]
+    
+    def _text_similarity(self, text1: str, text2: str) -> float:
+        """Calculate simple text similarity based on word overlap"""
+        import re
+        
+        # Normalize texts - remove punctuation and convert to lowercase
+        text1_clean = re.sub(r'[^\w\s]', ' ', text1.lower().strip())
+        text2_clean = re.sub(r'[^\w\s]', ' ', text2.lower().strip())
+        
+        # SPECIAL CASE: If database question is very short (1-2 words), check if ANY of those words appear in query
+        db_words = text2_clean.split()
+        if len(db_words) <= 2:
+            # For short database questions like "Executive Summary"
+            for word in db_words:
+                if len(word) > 3 and word in text1_clean:  # Check if significant word appears in query
+                    # Check if it's about same topic using keyword matching
+                    vendor_keywords = ['executive', 'summary', 'vendor', 'organization', 'company', 'business', 'about', 'information', 'general']
+                    recruiting_keywords = ['recruiting', 'recruitment', 'candidate', 'interview', 'hiring', 'process', 'evaluate']
+                    
+                    query_vendor_matches = sum(1 for kw in vendor_keywords if kw in text1_clean)
+                    db_vendor_matches = sum(1 for kw in vendor_keywords if kw in text2_clean)
+                    query_recruiting_matches = sum(1 for kw in recruiting_keywords if kw in text1_clean)
+                    
+                    # If db is "executive summary" and query has vendor/org keywords, VERY high match
+                    if ('executive' in text2_clean or 'summary' in text2_clean) and query_vendor_matches >= 2:
+                        print(f"✓ SHORT QUESTION MATCH: '{text2_clean}' matches vendor org query. Score: 0.98")
+                        return 0.98
+                    
+                    # If db is "executive summary" and query has even 1 vendor keyword, still good match
+                    if ('executive' in text2_clean or 'summary' in text2_clean) and query_vendor_matches >= 1 and query_recruiting_matches == 0:
+                        print(f"✓ EXECUTIVE SUMMARY match with vendor keywords. Score: 0.95")
+                        return 0.95
+                    
+                    # Don't match if query is about recruiting and db is executive summary
+                    if ('executive' in text2_clean or 'summary' in text2_clean) and query_recruiting_matches >= 2 and query_vendor_matches < 2:
+                        print(f"✗ Topic mismatch: Executive Summary vs Recruiting question")
+                        return 0.0
+        
+        # Check for keyword-based semantic matches first
+        keyword_groups = {
+            'vendor_info': ['executive', 'summary', 'vendor', 'organization', 'company', 'business', 'focus', 'specialty', 'employees', 'general', 'information', 'about'],
+            'recruiting': ['recruiting', 'recruitment', 'hiring', 'candidate', 'interview'],
+            'quality': ['quality', 'assurance', 'satisfaction', 'success', 'rate'],
+            'response_time': ['response', 'time', 'acknowledge', 'initial', 'mean', 'timeframe'],
+            'turnover': ['turnover', 'replacement', 'contract', 'period'],
+        }
+        
+        # Check if both texts belong to same semantic group
+        for group_name, keywords in keyword_groups.items():
+            text1_matches = sum(1 for kw in keywords if kw in text1_clean)
+            text2_matches = sum(1 for kw in keywords if kw in text2_clean)
+            
+            # Debug logging for vendor_info
+            if group_name == 'vendor_info':
+                print(f"\n=== VENDOR_INFO MATCHING DEBUG ===")
+                print(f"Text1 (query): '{text1_clean}'")
+                print(f"Text2 (db): '{text2_clean}'")
+                print(f"Text1 matches: {text1_matches}")
+                print(f"Text2 matches: {text2_matches}")
+                print(f"'executive' in text2: {'executive' in text2_clean}")
+                print(f"'summary' in text2: {'summary' in text2_clean}")
+                print(f"Condition 1 (query has 2+ vendor keywords AND db has executive/summary): {text1_matches >= 2 and ('executive' in text2_clean or 'summary' in text2_clean)}")
+                print(f"Condition 2 (db has 1+ vendor keyword AND query has executive/summary): {text2_matches >= 1 and ('executive' in text1_clean or 'summary' in text1_clean)}")
+            
+            # Special handling for vendor_info - match "Executive Summary" with organization questions
+            if group_name == 'vendor_info':
+                # If query has vendor/org keywords (2+) and db has "executive" or "summary"
+                if text1_matches >= 2 and ('executive' in text2_clean or 'summary' in text2_clean):
+                    print(f"✓ Keyword match: Vendor info question matched with Executive Summary. Score: 0.92")
+                    return 0.92
+                # If db has vendor keywords and query has "executive" or "summary"
+                if text2_matches >= 1 and ('executive' in text1_clean or 'summary' in text1_clean):
+                    print(f"✓ Keyword match: Executive Summary matched with vendor info. Score: 0.92")
+                    return 0.92
+                # Both have multiple vendor keywords
+                if text1_matches >= 3 and text2_matches >= 1:
+                    print(f"✓ Keyword match: Strong vendor/company information match. Score: 0.88")
+                    return 0.88
+            # For other groups, require matches in both
+            elif text1_matches >= 2 and text2_matches >= 1:
+                print(f"Keyword match: {group_name} group. Score: 0.80")
+                return 0.80
+        
+        # Check for substring match (very high score)
+        if text1_clean in text2_clean or text2_clean in text1_clean:
+            print(f"Substring match found! Score: 0.95")
+            return 0.95
+        
+        # Split into words
+        words1 = set(text1_clean.split())
+        words2 = set(text2_clean.split())
+        
+        # Remove only very common words (not content words like organization, company)
+        common_words = {'the', 'a', 'an', 'is', 'are', 'what', 'how', 'why', 'when', 'where', 'who', 'of', 'to', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'whats', 'your', 'our', 'their', 's', 'and', 'or'}
+        words1_filtered = words1 - common_words
+        words2_filtered = words2 - common_words
+        
+        print(f"Question: '{text1_clean}'")
+        print(f"Database: '{text2_clean}'")
+        print(f"Words1 filtered: {words1_filtered}")
+        print(f"Words2 filtered: {words2_filtered}")
+        
+        if len(words1_filtered) == 0 or len(words2_filtered) == 0:
+            # Fall back to unfiltered if all words were common
+            words1_filtered = words1
+            words2_filtered = words2
+        
+        if len(words1_filtered) == 0 or len(words2_filtered) == 0:
+            return 0.0
+        
+        # Calculate intersection
+        intersection = words1_filtered.intersection(words2_filtered)
+        print(f"Intersection: {intersection}")
+        
+        # If no meaningful words match, return 0
+        if len(intersection) == 0:
+            return 0.0
+        
+        # Calculate similarity based on smaller set (more lenient)
+        smaller_set_size = min(len(words1_filtered), len(words2_filtered))
+        match_ratio = len(intersection) / smaller_set_size
+        print(f"Match ratio: {match_ratio} ({len(intersection)}/{smaller_set_size})")
+        
+        # Boost if all important words from smaller question match
+        if match_ratio >= 0.5:  # If 50% or more words match
+            # High confidence
+            score = min(0.85 + (match_ratio * 0.15), 1.0)
+            print(f"High confidence score: {score}")
+            return score
+        elif match_ratio >= 0.3:  # If 30-50% words match
+            # Medium confidence
+            score = 0.65 + (match_ratio * 0.2)
+            print(f"Medium confidence score: {score}")
+            return score
+        else:
+            # Low confidence
+            score = match_ratio * 0.6
+            print(f"Low confidence score: {score}")
+            return score
+    
+    def _cosine_similarity(self, vec1: np.ndarray, vec2: np.ndarray) -> float:
+        """Calculate cosine similarity between two vectors"""
+        dot_product = np.dot(vec1, vec2)
+        norm1 = np.linalg.norm(vec1)
+        norm2 = np.linalg.norm(vec2)
+        
+        if norm1 == 0 or norm2 == 0:
+            return 0.0
+        
+        return dot_product / (norm1 * norm2)
+    
+    def check_exact_match(self, db: Session, question_text: str) -> Question:
+        """Check for exact question match (case-insensitive, trimmed)"""
+        # Normalize the input question
+        normalized_question = question_text.strip().lower()
+        
+        # Try exact match first
+        exact = db.query(Question).filter(
+            Question.question_text.ilike(normalized_question)
+        ).first()
+        
+        if exact:
+            return exact
+        
+        # Try with wildcards for very close matches (90%+ similar)
+        all_questions = db.query(Question).all()
+        for q in all_questions:
+            if q.question_text.strip().lower() == normalized_question:
+                return q
+        
+        return None
+
+similarity_service = SimilaritySearchService()

backend/requirements.txt

@@ -0,0 +1,11 @@
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+sqlalchemy==2.0.23
+python-multipart==0.0.6
+pdfplumber==0.10.3
+sentence-transformers==2.2.2
+numpy==1.24.3
+python-dotenv==1.0.0
+pydantic==2.5.0
+pydantic-settings==2.1.0
+requests==2.31.0