Download app.py from OrganizingForChange/WikiInterrogator: direct link, hf CLI and curl.
- Browser
- Download file 12.3 kB
-
https://huggingface.co/spaces/OrganizingForChange/WikiInterrogator/resolve/main/app.py
- Command line
-
hf download hf://spaces/OrganizingForChange/WikiInterrogator/app.py
-
curl -L -o app.py https://huggingface.co/spaces/OrganizingForChange/WikiInterrogator/resolve/main/app.py
12.3 kB
| import gradio as gr | |
| import os | |
| from typing import List, Tuple | |
| import requests | |
| from bs4 import BeautifulSoup | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain_openai import OpenAIEmbeddings, ChatOpenAI | |
| from langchain_community.vectorstores import Chroma | |
| from langchain_core.prompts import ChatPromptTemplate | |
| from langchain_core.runnables import RunnablePassthrough | |
| from langchain_core.output_parsers import StrOutputParser | |
| # Global variables to store state | |
| vector_store = None | |
| qa_chain = None | |
| wiki_url = None | |
| api_key = None | |
| def fetch_wiki_content(wiki_url: str) -> List[str]: | |
| """Fetch content from a wiki URL.""" | |
| try: | |
| response = requests.get(wiki_url, timeout=10) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| # Extract text content | |
| # Remove script and style elements | |
| for script in soup(["script", "style"]): | |
| script.decompose() | |
| # Get text | |
| text = soup.get_text() | |
| # Clean up text | |
| lines = (line.strip() for line in text.splitlines()) | |
| chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) | |
| text = '\n'.join(chunk for chunk in chunks if chunk) | |
| return [text] | |
| except Exception as e: | |
| raise Exception(f"Error fetching wiki content: {str(e)}") | |
| def fetch_wiki_pages(wiki_url: str, max_pages: int = 20) -> List[str]: | |
| """Fetch multiple pages from a wiki by following links.""" | |
| pages_content = [] | |
| visited = set() | |
| links_to_visit = [] | |
| try: | |
| # Start with the main page | |
| response = requests.get(wiki_url, timeout=10) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| # Get main page content | |
| for script in soup(["script", "style"]): | |
| script.decompose() | |
| text = soup.get_text() | |
| lines = (line.strip() for line in text.splitlines()) | |
| chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) | |
| text = '\n'.join(chunk for chunk in chunks if chunk) | |
| if text: | |
| pages_content.append(text) | |
| visited.add(wiki_url) | |
| # Collect all links from the main page | |
| base_url = '/'.join(wiki_url.split('/')[:3]) | |
| links = soup.find_all('a', href=True) | |
| for link in links: | |
| href = link.get('href') | |
| if not href: | |
| continue | |
| # Build full URL | |
| if href.startswith('/'): | |
| full_url = base_url + href | |
| elif href.startswith('http'): | |
| # Only include links from the same domain | |
| if base_url in href: | |
| full_url = href | |
| else: | |
| continue | |
| else: | |
| continue | |
| # Skip if already visited or in queue | |
| if full_url not in visited and full_url not in links_to_visit: | |
| # Filter out common non-content links | |
| skip_patterns = ['#', 'javascript:', 'mailto:', '.pdf', '.jpg', '.png', '.gif', | |
| 'login', 'logout', 'register', 'search', 'edit', 'history'] | |
| if not any(pattern in href.lower() for pattern in skip_patterns): | |
| links_to_visit.append(full_url) | |
| # Continue fetching pages until we reach max_pages or run out of links | |
| while len(pages_content) < max_pages and links_to_visit: | |
| current_url = links_to_visit.pop(0) | |
| if current_url in visited: | |
| continue | |
| try: | |
| page_response = requests.get(current_url, timeout=10, allow_redirects=True) | |
| if page_response.status_code == 200: | |
| page_soup = BeautifulSoup(page_response.content, 'html.parser') | |
| for script in page_soup(["script", "style", "nav", "footer", "header"]): | |
| script.decompose() | |
| page_text = page_soup.get_text() | |
| lines = (line.strip() for line in page_text.splitlines()) | |
| chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) | |
| page_text = '\n'.join(chunk for chunk in chunks if chunk) | |
| # Only add if there's substantial content | |
| if page_text and len(page_text.strip()) > 100: | |
| pages_content.append(page_text) | |
| visited.add(current_url) | |
| # Optionally collect more links from this page if we need more pages | |
| if len(pages_content) < max_pages: | |
| page_links = page_soup.find_all('a', href=True) | |
| for link in page_links: | |
| href = link.get('href') | |
| if not href: | |
| continue | |
| if href.startswith('/'): | |
| new_url = base_url + href | |
| elif href.startswith('http') and base_url in href: | |
| new_url = href | |
| else: | |
| continue | |
| if new_url not in visited and new_url not in links_to_visit: | |
| skip_patterns = ['#', 'javascript:', 'mailto:', '.pdf', '.jpg', | |
| '.png', '.gif', 'login', 'logout', 'register', | |
| 'search', 'edit', 'history'] | |
| if not any(pattern in href.lower() for pattern in skip_patterns): | |
| links_to_visit.append(new_url) | |
| except Exception as e: | |
| # Skip failed pages and continue | |
| continue | |
| return pages_content | |
| except Exception as e: | |
| raise Exception(f"Error fetching wiki pages: {str(e)}") | |
| def initialize_rag(wiki_url_input: str, api_key_input: str, max_pages: int = 20, progress=gr.Progress()) -> str: | |
| """Initialize the RAG system with wiki content.""" | |
| global vector_store, qa_chain, wiki_url, api_key | |
| if not wiki_url_input: | |
| return "β Please enter a wiki URL" | |
| if not api_key_input: | |
| return "β Please enter an API key" | |
| try: | |
| progress(0.1, desc="Setting up API key...") | |
| api_key = api_key_input | |
| wiki_url = wiki_url_input | |
| os.environ["OPENAI_API_KEY"] = api_key | |
| progress(0.2, desc="Fetching wiki content...") | |
| # Fetch wiki pages (convert to int in case it comes as float from Number input) | |
| documents = fetch_wiki_pages(wiki_url_input, max_pages=int(max_pages)) | |
| if not documents: | |
| return "β Failed to fetch wiki content. Please check the URL." | |
| progress(0.4, desc="Splitting documents into chunks...") | |
| # Split documents into chunks | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=1000, | |
| chunk_overlap=200, | |
| length_function=len, | |
| ) | |
| texts = text_splitter.create_documents(documents) | |
| progress(0.6, desc="Creating embeddings...") | |
| # Create embeddings | |
| embeddings = OpenAIEmbeddings(openai_api_key=api_key) | |
| progress(0.8, desc="Building vector store...") | |
| # Create vector store | |
| vector_store = Chroma.from_documents( | |
| documents=texts, | |
| embedding=embeddings, | |
| persist_directory="./chroma_db" | |
| ) | |
| progress(0.9, desc="Setting up QA chain...") | |
| # Create QA chain using newer LangChain API | |
| llm = ChatOpenAI(temperature=0, openai_api_key=api_key, model_name="gpt-3.5-turbo") | |
| # Create prompt template | |
| prompt = ChatPromptTemplate.from_template("""Use the following pieces of context to answer the question at the end. | |
| If you don't know the answer, just say that you don't know, don't try to make up an answer. | |
| Use three sentences maximum and keep the answer concise. | |
| Context: {context} | |
| Question: {question} | |
| Answer:""") | |
| # Create retriever | |
| retriever = vector_store.as_retriever(search_kwargs={"k": 3}) | |
| # Create the chain | |
| def format_docs(docs): | |
| return "\n\n".join(doc.page_content for doc in docs) | |
| qa_chain = ( | |
| {"context": retriever | format_docs, "question": RunnablePassthrough()} | |
| | prompt | |
| | llm | |
| | StrOutputParser() | |
| ) | |
| progress(1.0, desc="Ready!") | |
| return f"β Successfully loaded wiki from {wiki_url_input} ({len(documents)} pages)" | |
| except Exception as e: | |
| return f"β Error initializing RAG: {str(e)}" | |
| def chat_with_docs(message: str, history: List[List[str]]) -> Tuple[str, List[List[str]]]: | |
| """Handle chat messages with the RAG system.""" | |
| global qa_chain | |
| if qa_chain is None: | |
| return "β οΈ Please initialize the RAG system first by entering a wiki URL and API key, then clicking 'Initialize RAG'.", history | |
| try: | |
| # Get response from QA chain | |
| answer = qa_chain.invoke(message) | |
| # Add to history | |
| history.append([message, answer]) | |
| return "", history | |
| except Exception as e: | |
| error_msg = f"β Error: {str(e)}" | |
| history.append([message, error_msg]) | |
| return "", history | |
| # Create Gradio interface | |
| with gr.Blocks(title="Chat with Wiki Documents") as app: | |
| gr.Markdown("# π Chat with Wiki Documents") | |
| gr.Markdown("Enter your wiki URL and API key to start chatting with your documents!") | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| wiki_url_input = gr.Textbox( | |
| label="Wiki URL", | |
| placeholder="https://example.com/wiki", | |
| value="" | |
| ) | |
| with gr.Column(scale=2): | |
| api_key_input = gr.Textbox( | |
| label="OpenAI API Key", | |
| placeholder="sk-...", | |
| type="password", | |
| value="" | |
| ) | |
| with gr.Column(scale=1): | |
| max_pages_input = gr.Number( | |
| label="Max Pages", | |
| value=20, | |
| minimum=1, | |
| maximum=100, | |
| step=1, | |
| info="Number of pages to fetch" | |
| ) | |
| with gr.Row(): | |
| init_btn = gr.Button("Initialize RAG", variant="primary", scale=1) | |
| status_output = gr.Textbox( | |
| label="Status", | |
| interactive=False, | |
| value="Waiting for initialization..." | |
| ) | |
| gr.Markdown("---") | |
| chatbot = gr.Chatbot( | |
| label="Chat", | |
| height=500, | |
| show_copy_button=True | |
| ) | |
| with gr.Row(): | |
| msg_input = gr.Textbox( | |
| label="Message", | |
| placeholder="Ask a question about the wiki...", | |
| scale=4, | |
| container=False | |
| ) | |
| submit_btn = gr.Button("Send", variant="primary", scale=1) | |
| # Event handlers | |
| init_btn.click( | |
| fn=initialize_rag, | |
| inputs=[wiki_url_input, api_key_input, max_pages_input], | |
| outputs=status_output | |
| ) | |
| submit_btn.click( | |
| fn=chat_with_docs, | |
| inputs=[msg_input, chatbot], | |
| outputs=[msg_input, chatbot] | |
| ) | |
| msg_input.submit( | |
| fn=chat_with_docs, | |
| inputs=[msg_input, chatbot], | |
| outputs=[msg_input, chatbot] | |
| ) | |
| if __name__ == "__main__": | |
| app.launch() | |