Delete pipeline

pipeline/answer_generator.py

@@ -1,141 +0,0 @@
-
-import streamlit as st
-import google.generativeai as genai
-from collections import defaultdict
-import math
-
-def score_documents_with_gemini(docs, user_query):
-    model = genai.GenerativeModel("gemini-2.0-pro-exp-02-05")
-    scored_docs = []
-
-    for i, doc in enumerate(docs):
-        url = doc.metadata.get("url", "No URL")
-        snippet = doc.page_content[:500]
-
-        prompt = f"""
-            You are a relevance evaluator assistant.
-
-            Given the user's query and a document snippet, score how relevant this document is (1 to 5 scale):
-
-            User Query:
-            \"\"\"{user_query}\"\"\"
-
-            Document Snippet:
-            \"\"\"{snippet}\"\"\"
-
-            Return a single number (1-5):
-        """
-        try:
-            response = model.generate_content(prompt)
-            score = int("".join(filter(str.isdigit, response.text.strip())))
-        except:
-            score = 3  # fallback
-
-        scored_docs.append({
-            "doc": doc,
-            "url": url,
-            "score": score
-        })
-
-    scored_docs.sort(key=lambda x: x["score"], reverse=True)
-    return scored_docs
-
-def get_diverse_documents(scored_docs, min_required):
-    by_url = defaultdict(list)
-    for entry in scored_docs:
-        by_url[entry["url"]].append(entry)
-
-    diverse_docs = []
-    for entries in by_url.values():
-        if len(diverse_docs) >= min_required:
-            break
-        diverse_docs.append(entries[0])  # take top scoring for each unique URL
-
-    return diverse_docs
-
-def generate_answer(user_query, retriever, scraped_results):
-    raw_docs = retriever.get_relevant_documents(user_query)
-    scored_docs = score_documents_with_gemini(raw_docs, user_query)
-
-    min_required = max(1, int(0.7 * len(scraped_results)))
-    selected_docs = get_diverse_documents(scored_docs, min_required=math.ceil(0.7 * len(scraped_results)))
-
-
-    context = ""
-    unique_sources = {}
-    for i, entry in enumerate(selected_docs):
-        doc = entry["doc"]
-        url = doc.metadata.get("url", "No URL Provided")
-        if url not in unique_sources:
-            unique_sources[url] = f"Source {len(unique_sources) + 1}"
-        label = unique_sources[url]
-        context += f"[{label}] ({url}):\n{doc.page_content}\n\n"
-
-    prompt = f"""
-        You are a senior research analyst generating professional and comprehensive reports based on provided source documents.
-
-        Please create a well-structured response to the user query using the information below. Your goal is to present both qualitative and quantitative data (including tables and pricing breakdowns) in clean, readable Markdown.
-
-        ### Instructions:
-        - Use markdown headings for sections.
-        - Format numbers or pricing data in tables if applicable.
-        - Cite facts using inline citations like [Source 1].
-        - Use bullet points where useful.
-        - If structured data (e.g., features, specs, pricing, tables) is mentioned in sources, render it **as a table** with proper headers.
-        - You MUST incorporate information from **at least 3 different sources** in your response.
-        - If some sources contain overlapping content, reference each one explicitly.
-        - Avoid relying only on a single source unless it's the only one that contains all details.
-        - Use inline citations like [Source 1], [Source 2], etc. after each fact or quote.
-        - If you cannot extract anything unique from a source, say so in the "Source Coverage" section at the bottom.
-
-        ### Output Format:
-        - Clean, structured Markdown
-        - Use headers like `## Overview`, `## Key Pricing Details`, `## Comparison`, etc.
-        - Always include citations and distinguish when info comes from different sources.
-
-        ## Source Coverage
-        - Source 1: Used in Model Comparison and Free Usage info
-        - Source 2: Confirmed pricing differences
-        - Source 3: Mentioned rate limits and billing tags
-        - Source 4: Not used due to repetition with Source 1
- 
-        ---
-
-        ### SOURCE DOCUMENTS:
-        {context}
-
-        ---
-
-        ### USER QUERY:
-        \"\"\"{user_query}\"\"\"
-
-        Please begin.
-
-
-    """
-
-    model = genai.GenerativeModel("gemini-2.0-pro-exp-02-05")
-    response = model.generate_content(prompt)
-    answer = response.text.strip()
-
-    st.markdown("### 🧠 Answer")
-    if any(r.get("page", 1) > 1 for r in scraped_results):
-        st.markdown("<span style='color:orange; font-weight:bold;'>⚠️ Includes content from deeper Google search pages</span>", unsafe_allow_html=True)
-    st.markdown(answer, unsafe_allow_html=True)
-
-    # Relevance Table UI
-    with st.expander("📊 Document Relevance Ranking"):
-        for entry in scored_docs:
-            url = entry["url"]
-            score = entry["score"]
-            label = unique_sources.get(url, "-")
-            rating = "High" if score >= 4 else "Medium" if score == 3 else "Low"
-            st.markdown(f"<small>🔹 **{label}** | {rating} Relevance | Score: {score}/5 — <a href='{url}' target='_blank'>{url}</a></small>", unsafe_allow_html=True)
-
-    # Final Source List
-    with st.expander("🔗 Source Citations"):
-        if unique_sources:
-            for url, label in unique_sources.items():
-                st.markdown(f"<small>🔹 **{label}**: <a href='{url}' target='_blank'>{url}</a></small>", unsafe_allow_html=True)
-        else:
-            st.markdown("<small>No sources available.</small>", unsafe_allow_html=True)

pipeline/crawler.py

@@ -1,58 +0,0 @@
-# pipeline/crawler.py
-
-import requests
-from bs4 import BeautifulSoup
-from urllib.parse import urljoin, urlparse
-from typing import List, Dict, Tuple, Set
-from pipeline.link_ranker import rank_links_by_query
-
-def extract_internal_links(soup: BeautifulSoup, base_url: str) -> List[str]:
-    internal_links = []
-    base_domain = urlparse(base_url).netloc
-
-    for a in soup.find_all("a", href=True):
-        href = a["href"]
-        full_url = urljoin(base_url, href)
-        parsed = urlparse(full_url)
-
-        if parsed.netloc == base_domain:
-            internal_links.append(full_url)
-
-    return list(set(internal_links))
-
-def crawl_site(start_url: str, user_query: str, max_depth: int = 2, max_links: int = 5) -> List[Dict]:
-    visited: Set[str] = set()
-    queue: List[Tuple[str, int]] = [(start_url, 0)]
-    crawled_pages = []
-
-    while queue and len(crawled_pages) < max_links:
-        url, depth = queue.pop(0)
-        if url in visited or depth > max_depth:
-            continue
-        visited.add(url)
-
-        try:
-            response = requests.get(url, timeout=10)
-            response.raise_for_status()
-            html = response.text
-            soup = BeautifulSoup(html, "html.parser")
-            content = "\n".join(p.get_text() for p in soup.find_all("p") if p.get_text())
-
-            crawled_pages.append({
-                "url": url,
-                "depth": depth,
-                "content": content.strip()
-            })
-
-            if len(crawled_pages) >= max_links:
-                break
-
-            internal_links = extract_internal_links(soup, url)
-            ranked_links = rank_links_by_query(internal_links, user_query, top_k=5)
-            queue.extend([(link, depth + 1) for link in ranked_links if link not in visited])
-
-        except Exception as e:
-            print(f"[Crawler] Failed to crawl {url}: {e}")
-            continue
-
-    return crawled_pages

pipeline/embed_and_store.py

@@ -1,22 +0,0 @@
-
-from langchain.schema import Document
-from langchain.vectorstores import Chroma
-
-def embed_and_store_chunks(all_chunks, embedding_model, persist_dir="chroma_store"):
-    embedded_chunks = []
-    for chunk in all_chunks:
-        vector = embedding_model.embed_query(chunk["content"])
-        embedded_chunks.append({
-            "embedding": vector,
-            "content": chunk["content"],
-            "metadata": chunk["metadata"]
-        })
-
-    documents = [Document(page_content=chunk["content"], metadata=chunk["metadata"]) for chunk in embedded_chunks]
-    chroma_store = Chroma.from_documents(
-        documents=documents,
-        embedding=embedding_model,
-        persist_directory=persist_dir
-    )
-    chroma_store.persist()
-    return chroma_store

pipeline/link_ranker.py

@@ -1,32 +0,0 @@
-# pipeline/link_ranker.py
-
-from typing import List
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-
-def rank_links_by_query(links: List[str], query: str, top_k: int = 5) -> List[str]:
-    """
-    Rank internal links based on relevance to the query using TF-IDF cosine similarity.
-
-    Args:
-        links (List[str]): Anchor texts or hrefs to rank.
-        query (str): User's question or topic.
-        top_k (int): Number of top links to return.
-
-    Returns:
-        List[str]: Ranked list of links most relevant to the query.
-    """
-    if not links:
-        return []
-
-    try:
-        vectorizer = TfidfVectorizer().fit([query] + links)
-        vectors = vectorizer.transform([query] + links)
-        query_vec = vectors[0]
-        link_vecs = vectors[1:]
-        scores = cosine_similarity(query_vec, link_vecs).flatten()
-        ranked_links = [link for _, link in sorted(zip(scores, links), reverse=True)]
-        return ranked_links[:top_k]
-    except Exception as e:
-        print(f"[LinkRanker] Failed to rank links: {e}")
-        return links[:top_k]

pipeline/query_handler.py

@@ -1,8 +0,0 @@
-
-from agent.query_analyzer import QueryAnalyzer
-
-def analyze_query(user_query: str):
-    analyzer = QueryAnalyzer()
-    metadata = analyzer.analyze_query(user_query)
-    keyword_chunks = metadata.get("KeywordChunks", [[user_query]])
-    return keyword_chunks

pipeline/search_and_scrape.py

@@ -1,58 +0,0 @@
-
-# pipeline/search_and_scrape.py
-
-import streamlit as st
-from urllib.parse import urlparse
-from pipeline.crawler import crawl_site
-
-def search_and_scrape(keyword_chunks, search_tool, scraper, chunker, user_query, max_links=4, max_pages=3, max_crawl_depth=2, max_crawl_pages=3):
-    st.sidebar.markdown("### 🔗 Scraped Sources")
-    scraped_results = []
-    all_chunks = []
-
-    for chunk in keyword_chunks:
-        if len(scraped_results) >= max_links:
-            break
-
-        query = " ".join(chunk)
-        chunk_results = search_tool.search(query, num_results=10, max_pages=max_pages)
-
-        for result in chunk_results:
-            if len(scraped_results) >= max_links:
-                break
-
-            # Check if it's a homepage-style URL
-            parsed = urlparse(result["link"])
-            if parsed.path in ["", "/", "/index.html", "/home"]:
-                try:
-                    crawled_pages = crawl_site(result["link"], user_query, max_depth=max_crawl_depth, max_links=max_crawl_pages)
-                    for page in crawled_pages:
-                        if len(scraped_results) >= max_links:
-                            break
-                        doc_chunks = chunker.chunk_text(page["content"], {"url": page["url"], "title": f"Crawled from {result['link']}"})
-                        all_chunks.extend(doc_chunks)
-                        scraped_results.append({"title": f"Crawled from {result['link']}", "link": page["url"], "page": result.get("page", 1)})
-                        st.sidebar.markdown(f"<span style='color:green'>🌐 Crawled</span> 🔹 <a href='{page['url']}' target='_blank'>{page['url']}</a>", unsafe_allow_html=True)
-                except Exception as e:
-                    st.sidebar.warning(f"Failed to crawl {result['link']} — {e}")
-                continue
-
-            # Standard page scraping
-            scraped = scraper.scrape(result["link"])
-            if scraped["content"]:
-                doc_chunks = chunker.chunk_text(scraped["content"], {
-                    "url": result["link"],
-                    "title": result["title"]
-                })
-                all_chunks.extend(doc_chunks)
-                scraped_results.append(result)
-
-                page_info = result.get("page", 1)
-                if page_info > 1:
-                    st.sidebar.markdown(
-                        f"<span style='color:orange'>📄 Page {page_info}</span> 🔹 <a href='{result['link']}' target='_blank'>{result['title']}</a>",
-                        unsafe_allow_html=True)
-                else:
-                    st.sidebar.markdown(f"🔹 [Page {page_info}] [{result['title']}]({result['link']})")
-
-    return scraped_results, all_chunks