OrganizingForChange commited on
Commit
b8a537c
ยท
verified ยท
1 Parent(s): 2057f75

Upload 8 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ chroma_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,13 +1,66 @@
1
- ---
2
- title: WikiInterrogator
3
- emoji: ๐Ÿ˜ป
4
- colorFrom: gray
5
- colorTo: indigo
6
- sdk: gradio
7
- sdk_version: 5.49.1
8
- app_file: app.py
9
- pinned: false
10
- short_description: A tool to query a wiki page
11
- ---
12
-
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Chat with Wiki Documents
2
+
3
+ A Gradio application that allows you to chat with documents from any online wiki using Retrieval Augmented Generation (RAG).
4
+
5
+ ## Features
6
+
7
+ - ๐Ÿ”— Connect to any online wiki
8
+ - ๐Ÿ”‘ Secure API key input
9
+ - ๐Ÿ’ฌ Interactive chat interface
10
+ - ๐Ÿ“š Automatic document fetching and processing
11
+ - ๐Ÿ” Semantic search across wiki content
12
+ - ๐Ÿค– Powered by OpenAI's GPT models
13
+
14
+ ## Setup
15
+
16
+ 1. **Install dependencies:**
17
+ ```bash
18
+ pip install -r requirements.txt
19
+ ```
20
+
21
+ 2. **Get an OpenAI API key:**
22
+ - Sign up at [OpenAI](https://platform.openai.com/)
23
+ - Create an API key from your dashboard
24
+
25
+ 3. **Run the application:**
26
+ ```bash
27
+ python app.py
28
+ ```
29
+
30
+ 4. **Open your browser:**
31
+ - The app will be available at `http://localhost:7860`
32
+ - Or use the public URL provided in the terminal
33
+
34
+ ## Usage
35
+
36
+ 1. **Enter Wiki URL:** Paste the URL of the wiki you want to chat with
37
+ 2. **Enter API Key:** Paste your OpenAI API key
38
+ 3. **Click "Initialize RAG":** This will fetch and process the wiki content
39
+ 4. **Start Chatting:** Ask questions about the wiki content in the chat interface
40
+
41
+ ## How It Works
42
+
43
+ 1. **Document Fetching:** The app fetches content from the specified wiki URL and follows links to gather multiple pages
44
+ 2. **Text Processing:** Documents are split into chunks for efficient processing
45
+ 3. **Embedding:** Text chunks are converted to embeddings using OpenAI's embedding model
46
+ 4. **Vector Store:** Embeddings are stored in a ChromaDB vector database
47
+ 5. **Retrieval:** When you ask a question, relevant chunks are retrieved using semantic search
48
+ 6. **Generation:** The retrieved context is used to generate an answer using GPT
49
+
50
+ ## Notes
51
+
52
+ - The app fetches up to 10 wiki pages by default
53
+ - Vector database is stored locally in `./chroma_db`
54
+ - Your API key is only used for API calls and not stored permanently
55
+ - Make sure the wiki URL is publicly accessible
56
+
57
+ ## Troubleshooting
58
+
59
+ - **"Error fetching wiki content":** Check that the wiki URL is correct and publicly accessible
60
+ - **"Error initializing RAG":** Verify your API key is valid and has credits
61
+ - **Slow initialization:** Large wikis may take time to process. Be patient!
62
+
63
+ ## License
64
+
65
+ MIT
66
+
app.py ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import os
3
+ from typing import List, Tuple
4
+ import requests
5
+ from bs4 import BeautifulSoup
6
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
7
+ from langchain_openai import OpenAIEmbeddings, ChatOpenAI
8
+ from langchain_community.vectorstores import Chroma
9
+ from langchain_core.prompts import ChatPromptTemplate
10
+ from langchain_core.runnables import RunnablePassthrough
11
+ from langchain_core.output_parsers import StrOutputParser
12
+
13
+ # Global variables to store state
14
+ vector_store = None
15
+ qa_chain = None
16
+ wiki_url = None
17
+ api_key = None
18
+
19
+ def fetch_wiki_content(wiki_url: str) -> List[str]:
20
+ """Fetch content from a wiki URL."""
21
+ try:
22
+ response = requests.get(wiki_url, timeout=10)
23
+ response.raise_for_status()
24
+ soup = BeautifulSoup(response.content, 'html.parser')
25
+
26
+ # Extract text content
27
+ # Remove script and style elements
28
+ for script in soup(["script", "style"]):
29
+ script.decompose()
30
+
31
+ # Get text
32
+ text = soup.get_text()
33
+
34
+ # Clean up text
35
+ lines = (line.strip() for line in text.splitlines())
36
+ chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
37
+ text = '\n'.join(chunk for chunk in chunks if chunk)
38
+
39
+ return [text]
40
+ except Exception as e:
41
+ raise Exception(f"Error fetching wiki content: {str(e)}")
42
+
43
+ def fetch_wiki_pages(wiki_url: str, max_pages: int = 10) -> List[str]:
44
+ """Fetch multiple pages from a wiki by following links."""
45
+ pages_content = []
46
+ visited = set()
47
+
48
+ try:
49
+ # Start with the main page
50
+ response = requests.get(wiki_url, timeout=10)
51
+ response.raise_for_status()
52
+ soup = BeautifulSoup(response.content, 'html.parser')
53
+
54
+ # Get main page content
55
+ for script in soup(["script", "style"]):
56
+ script.decompose()
57
+ text = soup.get_text()
58
+ lines = (line.strip() for line in text.splitlines())
59
+ chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
60
+ text = '\n'.join(chunk for chunk in chunks if chunk)
61
+ pages_content.append(text)
62
+ visited.add(wiki_url)
63
+
64
+ # Try to find and follow wiki links
65
+ base_url = '/'.join(wiki_url.split('/')[:3])
66
+ links = soup.find_all('a', href=True)
67
+
68
+ for link in links[:max_pages-1]:
69
+ href = link.get('href')
70
+ if href and href.startswith('/'):
71
+ full_url = base_url + href
72
+ elif href and href.startswith('http'):
73
+ full_url = href
74
+ else:
75
+ continue
76
+
77
+ if full_url not in visited and len(pages_content) < max_pages:
78
+ try:
79
+ page_response = requests.get(full_url, timeout=5)
80
+ if page_response.status_code == 200:
81
+ page_soup = BeautifulSoup(page_response.content, 'html.parser')
82
+ for script in page_soup(["script", "style"]):
83
+ script.decompose()
84
+ page_text = page_soup.get_text()
85
+ lines = (line.strip() for line in page_text.splitlines())
86
+ chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
87
+ page_text = '\n'.join(chunk for chunk in chunks if chunk)
88
+ if page_text:
89
+ pages_content.append(page_text)
90
+ visited.add(full_url)
91
+ except:
92
+ continue
93
+
94
+ return pages_content
95
+ except Exception as e:
96
+ raise Exception(f"Error fetching wiki pages: {str(e)}")
97
+
98
+ def initialize_rag(wiki_url_input: str, api_key_input: str, progress=gr.Progress()) -> str:
99
+ """Initialize the RAG system with wiki content."""
100
+ global vector_store, qa_chain, wiki_url, api_key
101
+
102
+ if not wiki_url_input:
103
+ return "โŒ Please enter a wiki URL"
104
+ if not api_key_input:
105
+ return "โŒ Please enter an API key"
106
+
107
+ try:
108
+ progress(0.1, desc="Setting up API key...")
109
+ api_key = api_key_input
110
+ wiki_url = wiki_url_input
111
+ os.environ["OPENAI_API_KEY"] = api_key
112
+
113
+ progress(0.2, desc="Fetching wiki content...")
114
+ # Fetch wiki pages
115
+ documents = fetch_wiki_pages(wiki_url_input, max_pages=10)
116
+
117
+ if not documents:
118
+ return "โŒ Failed to fetch wiki content. Please check the URL."
119
+
120
+ progress(0.4, desc="Splitting documents into chunks...")
121
+ # Split documents into chunks
122
+ text_splitter = RecursiveCharacterTextSplitter(
123
+ chunk_size=1000,
124
+ chunk_overlap=200,
125
+ length_function=len,
126
+ )
127
+ texts = text_splitter.create_documents(documents)
128
+
129
+ progress(0.6, desc="Creating embeddings...")
130
+ # Create embeddings
131
+ embeddings = OpenAIEmbeddings(openai_api_key=api_key)
132
+
133
+ progress(0.8, desc="Building vector store...")
134
+ # Create vector store
135
+ vector_store = Chroma.from_documents(
136
+ documents=texts,
137
+ embedding=embeddings,
138
+ persist_directory="./chroma_db"
139
+ )
140
+
141
+ progress(0.9, desc="Setting up QA chain...")
142
+ # Create QA chain using newer LangChain API
143
+ llm = ChatOpenAI(temperature=0, openai_api_key=api_key, model_name="gpt-3.5-turbo")
144
+
145
+ # Create prompt template
146
+ prompt = ChatPromptTemplate.from_template("""Use the following pieces of context to answer the question at the end.
147
+ If you don't know the answer, just say that you don't know, don't try to make up an answer.
148
+ Use three sentences maximum and keep the answer concise.
149
+
150
+ Context: {context}
151
+
152
+ Question: {question}
153
+ Answer:""")
154
+
155
+ # Create retriever
156
+ retriever = vector_store.as_retriever(search_kwargs={"k": 3})
157
+
158
+ # Create the chain
159
+ def format_docs(docs):
160
+ return "\n\n".join(doc.page_content for doc in docs)
161
+
162
+ qa_chain = (
163
+ {"context": retriever | format_docs, "question": RunnablePassthrough()}
164
+ | prompt
165
+ | llm
166
+ | StrOutputParser()
167
+ )
168
+
169
+ progress(1.0, desc="Ready!")
170
+ return f"โœ… Successfully loaded wiki from {wiki_url_input} ({len(documents)} pages)"
171
+
172
+ except Exception as e:
173
+ return f"โŒ Error initializing RAG: {str(e)}"
174
+
175
+ def chat_with_docs(message: str, history: List[List[str]]) -> Tuple[str, List[List[str]]]:
176
+ """Handle chat messages with the RAG system."""
177
+ global qa_chain
178
+
179
+ if qa_chain is None:
180
+ return "โš ๏ธ Please initialize the RAG system first by entering a wiki URL and API key, then clicking 'Initialize RAG'.", history
181
+
182
+ try:
183
+ # Get response from QA chain
184
+ answer = qa_chain.invoke(message)
185
+
186
+ # Add to history
187
+ history.append([message, answer])
188
+ return "", history
189
+
190
+ except Exception as e:
191
+ error_msg = f"โŒ Error: {str(e)}"
192
+ history.append([message, error_msg])
193
+ return "", history
194
+
195
+ # Create Gradio interface
196
+ with gr.Blocks(title="Chat with Wiki Documents") as app:
197
+ gr.Markdown("# ๐Ÿ“š Chat with Wiki Documents")
198
+ gr.Markdown("Enter your wiki URL and API key to start chatting with your documents!")
199
+
200
+ with gr.Row():
201
+ with gr.Column(scale=2):
202
+ wiki_url_input = gr.Textbox(
203
+ label="Wiki URL",
204
+ placeholder="https://example.com/wiki",
205
+ value=""
206
+ )
207
+ with gr.Column(scale=2):
208
+ api_key_input = gr.Textbox(
209
+ label="OpenAI API Key",
210
+ placeholder="sk-...",
211
+ type="password",
212
+ value=""
213
+ )
214
+ with gr.Column(scale=1):
215
+ init_btn = gr.Button("Initialize RAG", variant="primary")
216
+
217
+ status_output = gr.Textbox(
218
+ label="Status",
219
+ interactive=False,
220
+ value="Waiting for initialization..."
221
+ )
222
+
223
+ gr.Markdown("---")
224
+
225
+ chatbot = gr.Chatbot(
226
+ label="Chat",
227
+ height=500,
228
+ show_copy_button=True
229
+ )
230
+
231
+ with gr.Row():
232
+ msg_input = gr.Textbox(
233
+ label="Message",
234
+ placeholder="Ask a question about the wiki...",
235
+ scale=4,
236
+ container=False
237
+ )
238
+ submit_btn = gr.Button("Send", variant="primary", scale=1)
239
+
240
+ # Event handlers
241
+ init_btn.click(
242
+ fn=initialize_rag,
243
+ inputs=[wiki_url_input, api_key_input],
244
+ outputs=status_output
245
+ )
246
+
247
+ submit_btn.click(
248
+ fn=chat_with_docs,
249
+ inputs=[msg_input, chatbot],
250
+ outputs=[msg_input, chatbot]
251
+ )
252
+
253
+ msg_input.submit(
254
+ fn=chat_with_docs,
255
+ inputs=[msg_input, chatbot],
256
+ outputs=[msg_input, chatbot]
257
+ )
258
+
259
+ if __name__ == "__main__":
260
+ app.launch(share=False, server_name="127.0.0.1", server_port=7860)
261
+
chroma_db/10118114-faf9-4197-b652-e9f3c9c22ddf/data_level0.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:537131aaf2e78bd9bba04c918a8c8f3442546005c5d94c9fbd852aff24ec5d1f
3
+ size 628400
chroma_db/10118114-faf9-4197-b652-e9f3c9c22ddf/header.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b081be2c2276a57e995075c7de2f3cb25e903798aac36d98042045533ab28f7d
3
+ size 100
chroma_db/10118114-faf9-4197-b652-e9f3c9c22ddf/length.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1ae1a029f259587ccebfa33dfdd33001144cadcc02a679f19ff96d033832b5e
3
+ size 400
chroma_db/10118114-faf9-4197-b652-e9f3c9c22ddf/link_lists.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
3
+ size 0
chroma_db/chroma.sqlite3 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a78de59be298d4285710b30dbd374e193223dfe59c892a8451f8b264ac5be6e8
3
+ size 729088
requirements.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ gradio>=4.0.0
2
+ langchain>=0.1.0
3
+ langchain-openai>=0.0.5
4
+ langchain-community>=0.0.20
5
+ openai>=1.0.0
6
+ chromadb>=0.4.0
7
+ beautifulsoup4>=4.12.0
8
+ requests>=2.31.0
9
+ tiktoken>=0.5.0
10
+