Spaces:
Configuration error
Configuration error
Clear all Space files
Browse files- .gitignore +0 -12
- .python-version +0 -1
- Dockerfile +0 -11
- README.md +0 -9
- app.py +0 -74
- main.py +0 -129
- mcp_server.py +0 -49
- pyproject.toml +0 -30
- requirements.txt +0 -23
- uv.lock +0 -0
.gitignore
DELETED
|
@@ -1,12 +0,0 @@
|
|
| 1 |
-
# Python-generated files
|
| 2 |
-
__pycache__/
|
| 3 |
-
*.py[oc]
|
| 4 |
-
build/
|
| 5 |
-
dist/
|
| 6 |
-
wheels/
|
| 7 |
-
*.egg-info
|
| 8 |
-
|
| 9 |
-
# Virtual environments
|
| 10 |
-
.venv
|
| 11 |
-
.env
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.python-version
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
3.12
|
|
|
|
|
|
Dockerfile
DELETED
|
@@ -1,11 +0,0 @@
|
|
| 1 |
-
FROM python:3.10
|
| 2 |
-
|
| 3 |
-
WORKDIR /app
|
| 4 |
-
|
| 5 |
-
COPY requirements.txt .
|
| 6 |
-
RUN pip install --no-cache-dir -r requirements.txt
|
| 7 |
-
|
| 8 |
-
COPY . .
|
| 9 |
-
|
| 10 |
-
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
---
|
| 2 |
-
title: RAGify Docs API
|
| 3 |
-
emoji: 🤖
|
| 4 |
-
colorFrom: blue
|
| 5 |
-
colorTo: purple
|
| 6 |
-
sdk: docker
|
| 7 |
-
app_file: app.py
|
| 8 |
-
pinned: false
|
| 9 |
-
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.py
DELETED
|
@@ -1,74 +0,0 @@
|
|
| 1 |
-
from fastapi.middleware.cors import CORSMiddleware
|
| 2 |
-
from fastapi.concurrency import run_in_threadpool
|
| 3 |
-
from fastapi import FastAPI, HTTPException
|
| 4 |
-
from pydantic import BaseModel, Field
|
| 5 |
-
from main import main
|
| 6 |
-
import os
|
| 7 |
-
|
| 8 |
-
# ========== Initialize FastAPI ==========
|
| 9 |
-
app = FastAPI(
|
| 10 |
-
title="RAGify Docs API",
|
| 11 |
-
description="A comprehensive API for RAGify Docs, allowing users to scrape documentation from a given URL and ask questions about it using a Retrieval-Augmented Generation (RAG) system built with LangChain and Groq.",
|
| 12 |
-
version="1.0.0"
|
| 13 |
-
)
|
| 14 |
-
|
| 15 |
-
# ========== Configure CORS ==========
|
| 16 |
-
app.add_middleware(
|
| 17 |
-
CORSMiddleware,
|
| 18 |
-
allow_origins=["*"], # Allow all origins
|
| 19 |
-
allow_credentials=True,
|
| 20 |
-
allow_methods=["*"], # Allow all methods
|
| 21 |
-
allow_headers=["*"], # Allow all headers
|
| 22 |
-
)
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
# schema for user input
|
| 27 |
-
class UserInput(BaseModel):
|
| 28 |
-
url: str = Field(..., description="The URL of the Documentation to scrape", examples=[
|
| 29 |
-
"https://docs.langchain.com/oss/python/langchain/overview"])
|
| 30 |
-
query: str = Field(...,
|
| 31 |
-
description="The question to ask about the scraped content", examples=["What is LangChain?"])
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
@app.get("/")
|
| 35 |
-
def home():
|
| 36 |
-
return {"message": "Welcome to the RAGify Docs API! Use the /ragify endpoint to ask questions about documentation."}
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
rag_cache = {}
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
@app.post("/ragify")
|
| 43 |
-
async def ragify(user_input: UserInput):
|
| 44 |
-
try:
|
| 45 |
-
url = user_input.url
|
| 46 |
-
|
| 47 |
-
# 🔥 CACHE + ASYNC SAFE
|
| 48 |
-
if url not in rag_cache:
|
| 49 |
-
rag_cache[url] = await run_in_threadpool(main, url)
|
| 50 |
-
|
| 51 |
-
rag_chain = rag_cache[url]
|
| 52 |
-
|
| 53 |
-
except Exception as e:
|
| 54 |
-
raise HTTPException(
|
| 55 |
-
status_code=500,
|
| 56 |
-
detail=f"RAG init failed: {str(e)}"
|
| 57 |
-
)
|
| 58 |
-
|
| 59 |
-
try:
|
| 60 |
-
response = await run_in_threadpool(
|
| 61 |
-
rag_chain.invoke,
|
| 62 |
-
{"input": user_input.query}
|
| 63 |
-
)
|
| 64 |
-
|
| 65 |
-
except Exception as e:
|
| 66 |
-
raise HTTPException(
|
| 67 |
-
status_code=500,
|
| 68 |
-
detail=f"Error invoking RAG chain: {str(e)}"
|
| 69 |
-
)
|
| 70 |
-
|
| 71 |
-
return {
|
| 72 |
-
"answer": response["answer"],
|
| 73 |
-
"sources": list(set(doc.metadata.get("source") for doc in response["context"]))
|
| 74 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
main.py
DELETED
|
@@ -1,129 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
RAGify Docs - A Developers Tool
|
| 3 |
-
A Developers Tool — Scrape entire documentation recursively and ask questions using AI
|
| 4 |
-
"""
|
| 5 |
-
|
| 6 |
-
from langchain_classic.chains.combine_documents import create_stuff_documents_chain
|
| 7 |
-
from langchain_community.document_loaders import RecursiveUrlLoader
|
| 8 |
-
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 9 |
-
from langchain_huggingface.embeddings import HuggingFaceEmbeddings
|
| 10 |
-
from langchain_core.vectorstores import InMemoryVectorStore
|
| 11 |
-
from langchain_classic.chains import create_retrieval_chain
|
| 12 |
-
from langchain_core.prompts import ChatPromptTemplate
|
| 13 |
-
from langchain_ollama import ChatOllama
|
| 14 |
-
from langchain_groq import ChatGroq
|
| 15 |
-
from dotenv import load_dotenv
|
| 16 |
-
from bs4 import BeautifulSoup
|
| 17 |
-
from tqdm import tqdm
|
| 18 |
-
import re
|
| 19 |
-
|
| 20 |
-
from bs4 import XMLParsedAsHTMLWarning
|
| 21 |
-
import warnings
|
| 22 |
-
|
| 23 |
-
warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
|
| 24 |
-
|
| 25 |
-
load_dotenv() # Load environment variables from .env file
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
def main(url: str):
|
| 29 |
-
# 1. SCRAPER SETUP
|
| 30 |
-
|
| 31 |
-
def bs4_extractor(html: str) -> str:
|
| 32 |
-
soup = BeautifulSoup(html, "lxml")
|
| 33 |
-
return re.sub(r"\n\n+", "\n\n", soup.text).strip()
|
| 34 |
-
|
| 35 |
-
website_url = url
|
| 36 |
-
loader = RecursiveUrlLoader(
|
| 37 |
-
website_url, extractor=bs4_extractor)
|
| 38 |
-
|
| 39 |
-
# 2. STREAMING & SPLITTING
|
| 40 |
-
|
| 41 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
| 42 |
-
chunk_size=1000, chunk_overlap=200)
|
| 43 |
-
|
| 44 |
-
all_chunks = []
|
| 45 |
-
|
| 46 |
-
# We use a loop to handle documents one-by-one (Streaming)
|
| 47 |
-
for doc in tqdm(loader.lazy_load(), bar_format="[INFO] {n_fmt} Pages Scraped."):
|
| 48 |
-
|
| 49 |
-
# split_documents returns a LIST of chunks for this specific page
|
| 50 |
-
chunks = text_splitter.split_documents([doc])
|
| 51 |
-
|
| 52 |
-
# We use EXTEND to flatten those chunks into our main list
|
| 53 |
-
all_chunks.extend(chunks)
|
| 54 |
-
|
| 55 |
-
print(f"[INFO] Created {len(all_chunks)} chunks.")
|
| 56 |
-
|
| 57 |
-
# 3. EMBEDDINGS & VECTOR STORE
|
| 58 |
-
|
| 59 |
-
print("[INFO] Embedding model initializing...")
|
| 60 |
-
|
| 61 |
-
embeddings = HuggingFaceEmbeddings(
|
| 62 |
-
model_name="sentence-transformers/all-MiniLM-L6-v2")
|
| 63 |
-
|
| 64 |
-
# Pass the list of chunks directly - metadata is preserved automatically!
|
| 65 |
-
vector_store = InMemoryVectorStore.from_documents(
|
| 66 |
-
documents=all_chunks,
|
| 67 |
-
embedding=embeddings
|
| 68 |
-
)
|
| 69 |
-
|
| 70 |
-
print("[INFO] Vector store created.")
|
| 71 |
-
|
| 72 |
-
# 4. RETRIEVER SETUP
|
| 73 |
-
|
| 74 |
-
retriever = vector_store.as_retriever(
|
| 75 |
-
search_type="mmr",
|
| 76 |
-
search_kwargs={
|
| 77 |
-
"k": 5,
|
| 78 |
-
|
| 79 |
-
"fetch_k": 10, # it gives 10 candidates, but only pick the 5 most diverse ones
|
| 80 |
-
|
| 81 |
-
"lambda_mult": 0.5 # 1.0 = Pure similarity, 0.0 = Pure diversity
|
| 82 |
-
}
|
| 83 |
-
)
|
| 84 |
-
|
| 85 |
-
# 5. LLM & PROMPT TEMPLATE
|
| 86 |
-
|
| 87 |
-
llm = ChatGroq(model="openai/gpt-oss-120b", temperature=0.2)
|
| 88 |
-
|
| 89 |
-
# The chain expects {context} and {input}
|
| 90 |
-
prompt = ChatPromptTemplate.from_template("""
|
| 91 |
-
You are a helpful and factual AI assistant.
|
| 92 |
-
Use the following retrieved context to answer the user's question.
|
| 93 |
-
|
| 94 |
-
If the answer is not found in the context, reply with:
|
| 95 |
-
"I'm not sure based on the provided information."
|
| 96 |
-
|
| 97 |
-
<context>
|
| 98 |
-
{context}
|
| 99 |
-
</context>
|
| 100 |
-
|
| 101 |
-
Question: {input}
|
| 102 |
-
""")
|
| 103 |
-
|
| 104 |
-
# 6. THE CHAIN
|
| 105 |
-
|
| 106 |
-
document_chain = create_stuff_documents_chain(llm, prompt)
|
| 107 |
-
rag_chain = create_retrieval_chain(retriever, document_chain)
|
| 108 |
-
|
| 109 |
-
return rag_chain
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
if __name__ == "__main__":
|
| 113 |
-
url = input("Enter the website URL to scrape: ")
|
| 114 |
-
rag_chain = main(url)
|
| 115 |
-
|
| 116 |
-
print("\n🚀 RAG System Ready! (Type '0' to exit)")
|
| 117 |
-
while True:
|
| 118 |
-
query = input("\nYou: ")
|
| 119 |
-
if query == "0":
|
| 120 |
-
break
|
| 121 |
-
|
| 122 |
-
# One call handles retrieval, prompt formatting, and LLM generation
|
| 123 |
-
response = rag_chain.invoke({"input": query})
|
| 124 |
-
|
| 125 |
-
print(f"\n🧠 AI: {response['answer']}")
|
| 126 |
-
|
| 127 |
-
# Optional: show which URLs were used
|
| 128 |
-
sources = {doc.metadata.get('source') for doc in response['context']}
|
| 129 |
-
print(f"📍 Sources: {', '.join(sources)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
mcp_server.py
DELETED
|
@@ -1,49 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
from fastmcp import FastMCP
|
| 3 |
-
from main import main
|
| 4 |
-
|
| 5 |
-
mcp = FastMCP(
|
| 6 |
-
name="RAGify Docs MCP Server"
|
| 7 |
-
)
|
| 8 |
-
|
| 9 |
-
rag_cache = {}
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
@mcp.tool(
|
| 13 |
-
name="ask_docs",
|
| 14 |
-
description="Ask questions about any documentation website by providing a URL and a question."
|
| 15 |
-
)
|
| 16 |
-
def ask_docs(url: str, query: str):
|
| 17 |
-
try:
|
| 18 |
-
# 🔥 Cache
|
| 19 |
-
if url not in rag_cache:
|
| 20 |
-
rag_cache[url] = main(url)
|
| 21 |
-
|
| 22 |
-
rag_chain = rag_cache[url]
|
| 23 |
-
|
| 24 |
-
response = rag_chain.invoke({"input": query})
|
| 25 |
-
|
| 26 |
-
# ✅ Return structured response
|
| 27 |
-
return {
|
| 28 |
-
"answer": response["answer"],
|
| 29 |
-
"sources": list(set(
|
| 30 |
-
doc.metadata.get("source")
|
| 31 |
-
for doc in response["context"]
|
| 32 |
-
if doc.metadata.get("source")
|
| 33 |
-
))
|
| 34 |
-
}
|
| 35 |
-
|
| 36 |
-
except Exception as e:
|
| 37 |
-
return {
|
| 38 |
-
"error": str(e)
|
| 39 |
-
}
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
if __name__ == "__main__":
|
| 43 |
-
port = int(os.environ.get("PORT", 8000))
|
| 44 |
-
|
| 45 |
-
mcp.run(
|
| 46 |
-
transport="streamable-http",
|
| 47 |
-
host="0.0.0.0",
|
| 48 |
-
port=port
|
| 49 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pyproject.toml
DELETED
|
@@ -1,30 +0,0 @@
|
|
| 1 |
-
[project]
|
| 2 |
-
name = "ragify-docs-api"
|
| 3 |
-
version = "0.1.0"
|
| 4 |
-
description = "Add your description here"
|
| 5 |
-
readme = "README.md"
|
| 6 |
-
requires-python = ">=3.12"
|
| 7 |
-
dependencies = [
|
| 8 |
-
"beautifulsoup4>=4.14.3",
|
| 9 |
-
"fastapi>=0.135.2",
|
| 10 |
-
"fastmcp>=3.1.1",
|
| 11 |
-
"hf-xet>=1.4.2",
|
| 12 |
-
"ipywidgets>=8.1.8",
|
| 13 |
-
"langchain>=1.2.13",
|
| 14 |
-
"langchain-classic>=1.0.3",
|
| 15 |
-
"langchain-community>=0.4.1",
|
| 16 |
-
"langchain-groq>=1.1.2",
|
| 17 |
-
"langchain-huggingface>=1.2.1",
|
| 18 |
-
"langchain-mcp-adapters>=0.2.2",
|
| 19 |
-
"langchain-ollama>=1.0.1",
|
| 20 |
-
"langchain-text-splitters>=1.1.1",
|
| 21 |
-
"langgraph>=1.1.3",
|
| 22 |
-
"lxml>=6.0.2",
|
| 23 |
-
"nest-asyncio>=1.6.0",
|
| 24 |
-
"pydantic>=2.12.5",
|
| 25 |
-
"python-dotenv>=1.2.2",
|
| 26 |
-
"sentence-transformers>=5.3.0",
|
| 27 |
-
"streamlit>=1.55.0",
|
| 28 |
-
"tqdm>=4.67.3",
|
| 29 |
-
"uvicorn>=0.42.0",
|
| 30 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
DELETED
|
@@ -1,23 +0,0 @@
|
|
| 1 |
-
pydantic
|
| 2 |
-
langchain
|
| 3 |
-
langchain-community
|
| 4 |
-
langgraph
|
| 5 |
-
python-dotenv
|
| 6 |
-
langchain-groq
|
| 7 |
-
langchain-ollama
|
| 8 |
-
streamlit
|
| 9 |
-
langchain-classic
|
| 10 |
-
langchain-text-splitters
|
| 11 |
-
beautifulsoup4
|
| 12 |
-
lxml
|
| 13 |
-
tqdm
|
| 14 |
-
ipywidgets
|
| 15 |
-
hf_xet
|
| 16 |
-
nest_asyncio
|
| 17 |
-
langchain-huggingface
|
| 18 |
-
sentence_transformers
|
| 19 |
-
fastapi
|
| 20 |
-
uvicorn
|
| 21 |
-
fastmcp
|
| 22 |
-
langchain-mcp-adapters
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
uv.lock
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|