Yashodeep2006 commited on
Commit
1faff3d
·
verified ·
1 Parent(s): c4809ad

Clear all Space files

Browse files
Files changed (10) hide show
  1. .gitignore +0 -12
  2. .python-version +0 -1
  3. Dockerfile +0 -11
  4. README.md +0 -9
  5. app.py +0 -74
  6. main.py +0 -129
  7. mcp_server.py +0 -49
  8. pyproject.toml +0 -30
  9. requirements.txt +0 -23
  10. uv.lock +0 -0
.gitignore DELETED
@@ -1,12 +0,0 @@
1
- # Python-generated files
2
- __pycache__/
3
- *.py[oc]
4
- build/
5
- dist/
6
- wheels/
7
- *.egg-info
8
-
9
- # Virtual environments
10
- .venv
11
- .env
12
-
 
 
 
 
 
 
 
 
 
 
 
 
 
.python-version DELETED
@@ -1 +0,0 @@
1
- 3.12
 
 
Dockerfile DELETED
@@ -1,11 +0,0 @@
1
- FROM python:3.10
2
-
3
- WORKDIR /app
4
-
5
- COPY requirements.txt .
6
- RUN pip install --no-cache-dir -r requirements.txt
7
-
8
- COPY . .
9
-
10
- CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
11
-
 
 
 
 
 
 
 
 
 
 
 
 
README.md DELETED
@@ -1,9 +0,0 @@
1
- ---
2
- title: RAGify Docs API
3
- emoji: 🤖
4
- colorFrom: blue
5
- colorTo: purple
6
- sdk: docker
7
- app_file: app.py
8
- pinned: false
9
- ---
 
 
 
 
 
 
 
 
 
 
app.py DELETED
@@ -1,74 +0,0 @@
1
- from fastapi.middleware.cors import CORSMiddleware
2
- from fastapi.concurrency import run_in_threadpool
3
- from fastapi import FastAPI, HTTPException
4
- from pydantic import BaseModel, Field
5
- from main import main
6
- import os
7
-
8
- # ========== Initialize FastAPI ==========
9
- app = FastAPI(
10
- title="RAGify Docs API",
11
- description="A comprehensive API for RAGify Docs, allowing users to scrape documentation from a given URL and ask questions about it using a Retrieval-Augmented Generation (RAG) system built with LangChain and Groq.",
12
- version="1.0.0"
13
- )
14
-
15
- # ========== Configure CORS ==========
16
- app.add_middleware(
17
- CORSMiddleware,
18
- allow_origins=["*"], # Allow all origins
19
- allow_credentials=True,
20
- allow_methods=["*"], # Allow all methods
21
- allow_headers=["*"], # Allow all headers
22
- )
23
-
24
-
25
-
26
- # schema for user input
27
- class UserInput(BaseModel):
28
- url: str = Field(..., description="The URL of the Documentation to scrape", examples=[
29
- "https://docs.langchain.com/oss/python/langchain/overview"])
30
- query: str = Field(...,
31
- description="The question to ask about the scraped content", examples=["What is LangChain?"])
32
-
33
-
34
- @app.get("/")
35
- def home():
36
- return {"message": "Welcome to the RAGify Docs API! Use the /ragify endpoint to ask questions about documentation."}
37
-
38
-
39
- rag_cache = {}
40
-
41
-
42
- @app.post("/ragify")
43
- async def ragify(user_input: UserInput):
44
- try:
45
- url = user_input.url
46
-
47
- # 🔥 CACHE + ASYNC SAFE
48
- if url not in rag_cache:
49
- rag_cache[url] = await run_in_threadpool(main, url)
50
-
51
- rag_chain = rag_cache[url]
52
-
53
- except Exception as e:
54
- raise HTTPException(
55
- status_code=500,
56
- detail=f"RAG init failed: {str(e)}"
57
- )
58
-
59
- try:
60
- response = await run_in_threadpool(
61
- rag_chain.invoke,
62
- {"input": user_input.query}
63
- )
64
-
65
- except Exception as e:
66
- raise HTTPException(
67
- status_code=500,
68
- detail=f"Error invoking RAG chain: {str(e)}"
69
- )
70
-
71
- return {
72
- "answer": response["answer"],
73
- "sources": list(set(doc.metadata.get("source") for doc in response["context"]))
74
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
main.py DELETED
@@ -1,129 +0,0 @@
1
- """
2
- RAGify Docs - A Developers Tool
3
- A Developers Tool — Scrape entire documentation recursively and ask questions using AI
4
- """
5
-
6
- from langchain_classic.chains.combine_documents import create_stuff_documents_chain
7
- from langchain_community.document_loaders import RecursiveUrlLoader
8
- from langchain_text_splitters import RecursiveCharacterTextSplitter
9
- from langchain_huggingface.embeddings import HuggingFaceEmbeddings
10
- from langchain_core.vectorstores import InMemoryVectorStore
11
- from langchain_classic.chains import create_retrieval_chain
12
- from langchain_core.prompts import ChatPromptTemplate
13
- from langchain_ollama import ChatOllama
14
- from langchain_groq import ChatGroq
15
- from dotenv import load_dotenv
16
- from bs4 import BeautifulSoup
17
- from tqdm import tqdm
18
- import re
19
-
20
- from bs4 import XMLParsedAsHTMLWarning
21
- import warnings
22
-
23
- warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
24
-
25
- load_dotenv() # Load environment variables from .env file
26
-
27
-
28
- def main(url: str):
29
- # 1. SCRAPER SETUP
30
-
31
- def bs4_extractor(html: str) -> str:
32
- soup = BeautifulSoup(html, "lxml")
33
- return re.sub(r"\n\n+", "\n\n", soup.text).strip()
34
-
35
- website_url = url
36
- loader = RecursiveUrlLoader(
37
- website_url, extractor=bs4_extractor)
38
-
39
- # 2. STREAMING & SPLITTING
40
-
41
- text_splitter = RecursiveCharacterTextSplitter(
42
- chunk_size=1000, chunk_overlap=200)
43
-
44
- all_chunks = []
45
-
46
- # We use a loop to handle documents one-by-one (Streaming)
47
- for doc in tqdm(loader.lazy_load(), bar_format="[INFO] {n_fmt} Pages Scraped."):
48
-
49
- # split_documents returns a LIST of chunks for this specific page
50
- chunks = text_splitter.split_documents([doc])
51
-
52
- # We use EXTEND to flatten those chunks into our main list
53
- all_chunks.extend(chunks)
54
-
55
- print(f"[INFO] Created {len(all_chunks)} chunks.")
56
-
57
- # 3. EMBEDDINGS & VECTOR STORE
58
-
59
- print("[INFO] Embedding model initializing...")
60
-
61
- embeddings = HuggingFaceEmbeddings(
62
- model_name="sentence-transformers/all-MiniLM-L6-v2")
63
-
64
- # Pass the list of chunks directly - metadata is preserved automatically!
65
- vector_store = InMemoryVectorStore.from_documents(
66
- documents=all_chunks,
67
- embedding=embeddings
68
- )
69
-
70
- print("[INFO] Vector store created.")
71
-
72
- # 4. RETRIEVER SETUP
73
-
74
- retriever = vector_store.as_retriever(
75
- search_type="mmr",
76
- search_kwargs={
77
- "k": 5,
78
-
79
- "fetch_k": 10, # it gives 10 candidates, but only pick the 5 most diverse ones
80
-
81
- "lambda_mult": 0.5 # 1.0 = Pure similarity, 0.0 = Pure diversity
82
- }
83
- )
84
-
85
- # 5. LLM & PROMPT TEMPLATE
86
-
87
- llm = ChatGroq(model="openai/gpt-oss-120b", temperature=0.2)
88
-
89
- # The chain expects {context} and {input}
90
- prompt = ChatPromptTemplate.from_template("""
91
- You are a helpful and factual AI assistant.
92
- Use the following retrieved context to answer the user's question.
93
-
94
- If the answer is not found in the context, reply with:
95
- "I'm not sure based on the provided information."
96
-
97
- <context>
98
- {context}
99
- </context>
100
-
101
- Question: {input}
102
- """)
103
-
104
- # 6. THE CHAIN
105
-
106
- document_chain = create_stuff_documents_chain(llm, prompt)
107
- rag_chain = create_retrieval_chain(retriever, document_chain)
108
-
109
- return rag_chain
110
-
111
-
112
- if __name__ == "__main__":
113
- url = input("Enter the website URL to scrape: ")
114
- rag_chain = main(url)
115
-
116
- print("\n🚀 RAG System Ready! (Type '0' to exit)")
117
- while True:
118
- query = input("\nYou: ")
119
- if query == "0":
120
- break
121
-
122
- # One call handles retrieval, prompt formatting, and LLM generation
123
- response = rag_chain.invoke({"input": query})
124
-
125
- print(f"\n🧠 AI: {response['answer']}")
126
-
127
- # Optional: show which URLs were used
128
- sources = {doc.metadata.get('source') for doc in response['context']}
129
- print(f"📍 Sources: {', '.join(sources)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mcp_server.py DELETED
@@ -1,49 +0,0 @@
1
- import os
2
- from fastmcp import FastMCP
3
- from main import main
4
-
5
- mcp = FastMCP(
6
- name="RAGify Docs MCP Server"
7
- )
8
-
9
- rag_cache = {}
10
-
11
-
12
- @mcp.tool(
13
- name="ask_docs",
14
- description="Ask questions about any documentation website by providing a URL and a question."
15
- )
16
- def ask_docs(url: str, query: str):
17
- try:
18
- # 🔥 Cache
19
- if url not in rag_cache:
20
- rag_cache[url] = main(url)
21
-
22
- rag_chain = rag_cache[url]
23
-
24
- response = rag_chain.invoke({"input": query})
25
-
26
- # ✅ Return structured response
27
- return {
28
- "answer": response["answer"],
29
- "sources": list(set(
30
- doc.metadata.get("source")
31
- for doc in response["context"]
32
- if doc.metadata.get("source")
33
- ))
34
- }
35
-
36
- except Exception as e:
37
- return {
38
- "error": str(e)
39
- }
40
-
41
-
42
- if __name__ == "__main__":
43
- port = int(os.environ.get("PORT", 8000))
44
-
45
- mcp.run(
46
- transport="streamable-http",
47
- host="0.0.0.0",
48
- port=port
49
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
pyproject.toml DELETED
@@ -1,30 +0,0 @@
1
- [project]
2
- name = "ragify-docs-api"
3
- version = "0.1.0"
4
- description = "Add your description here"
5
- readme = "README.md"
6
- requires-python = ">=3.12"
7
- dependencies = [
8
- "beautifulsoup4>=4.14.3",
9
- "fastapi>=0.135.2",
10
- "fastmcp>=3.1.1",
11
- "hf-xet>=1.4.2",
12
- "ipywidgets>=8.1.8",
13
- "langchain>=1.2.13",
14
- "langchain-classic>=1.0.3",
15
- "langchain-community>=0.4.1",
16
- "langchain-groq>=1.1.2",
17
- "langchain-huggingface>=1.2.1",
18
- "langchain-mcp-adapters>=0.2.2",
19
- "langchain-ollama>=1.0.1",
20
- "langchain-text-splitters>=1.1.1",
21
- "langgraph>=1.1.3",
22
- "lxml>=6.0.2",
23
- "nest-asyncio>=1.6.0",
24
- "pydantic>=2.12.5",
25
- "python-dotenv>=1.2.2",
26
- "sentence-transformers>=5.3.0",
27
- "streamlit>=1.55.0",
28
- "tqdm>=4.67.3",
29
- "uvicorn>=0.42.0",
30
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
requirements.txt DELETED
@@ -1,23 +0,0 @@
1
- pydantic
2
- langchain
3
- langchain-community
4
- langgraph
5
- python-dotenv
6
- langchain-groq
7
- langchain-ollama
8
- streamlit
9
- langchain-classic
10
- langchain-text-splitters
11
- beautifulsoup4
12
- lxml
13
- tqdm
14
- ipywidgets
15
- hf_xet
16
- nest_asyncio
17
- langchain-huggingface
18
- sentence_transformers
19
- fastapi
20
- uvicorn
21
- fastmcp
22
- langchain-mcp-adapters
23
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
uv.lock DELETED
The diff for this file is too large to render. See raw diff