Upload 8 files
Browse files
app.py
CHANGED
|
@@ -40,10 +40,11 @@ def fetch_wiki_content(wiki_url: str) -> List[str]:
|
|
| 40 |
except Exception as e:
|
| 41 |
raise Exception(f"Error fetching wiki content: {str(e)}")
|
| 42 |
|
| 43 |
-
def fetch_wiki_pages(wiki_url: str, max_pages: int =
|
| 44 |
"""Fetch multiple pages from a wiki by following links."""
|
| 45 |
pages_content = []
|
| 46 |
visited = set()
|
|
|
|
| 47 |
|
| 48 |
try:
|
| 49 |
# Start with the main page
|
|
@@ -58,44 +59,92 @@ def fetch_wiki_pages(wiki_url: str, max_pages: int = 10) -> List[str]:
|
|
| 58 |
lines = (line.strip() for line in text.splitlines())
|
| 59 |
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
| 60 |
text = '\n'.join(chunk for chunk in chunks if chunk)
|
| 61 |
-
|
| 62 |
-
|
|
|
|
| 63 |
|
| 64 |
-
#
|
| 65 |
base_url = '/'.join(wiki_url.split('/')[:3])
|
| 66 |
links = soup.find_all('a', href=True)
|
| 67 |
|
| 68 |
-
for link in links
|
| 69 |
href = link.get('href')
|
| 70 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
full_url = base_url + href
|
| 72 |
-
elif href
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
else:
|
| 75 |
continue
|
| 76 |
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
|
| 94 |
return pages_content
|
| 95 |
except Exception as e:
|
| 96 |
raise Exception(f"Error fetching wiki pages: {str(e)}")
|
| 97 |
|
| 98 |
-
def initialize_rag(wiki_url_input: str, api_key_input: str, progress=gr.Progress()) -> str:
|
| 99 |
"""Initialize the RAG system with wiki content."""
|
| 100 |
global vector_store, qa_chain, wiki_url, api_key
|
| 101 |
|
|
@@ -111,8 +160,8 @@ def initialize_rag(wiki_url_input: str, api_key_input: str, progress=gr.Progress
|
|
| 111 |
os.environ["OPENAI_API_KEY"] = api_key
|
| 112 |
|
| 113 |
progress(0.2, desc="Fetching wiki content...")
|
| 114 |
-
# Fetch wiki pages
|
| 115 |
-
documents = fetch_wiki_pages(wiki_url_input, max_pages=
|
| 116 |
|
| 117 |
if not documents:
|
| 118 |
return "❌ Failed to fetch wiki content. Please check the URL."
|
|
@@ -212,7 +261,17 @@ with gr.Blocks(title="Chat with Wiki Documents") as app:
|
|
| 212 |
value=""
|
| 213 |
)
|
| 214 |
with gr.Column(scale=1):
|
| 215 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
status_output = gr.Textbox(
|
| 218 |
label="Status",
|
|
@@ -240,7 +299,7 @@ with gr.Blocks(title="Chat with Wiki Documents") as app:
|
|
| 240 |
# Event handlers
|
| 241 |
init_btn.click(
|
| 242 |
fn=initialize_rag,
|
| 243 |
-
inputs=[wiki_url_input, api_key_input],
|
| 244 |
outputs=status_output
|
| 245 |
)
|
| 246 |
|
|
|
|
| 40 |
except Exception as e:
|
| 41 |
raise Exception(f"Error fetching wiki content: {str(e)}")
|
| 42 |
|
| 43 |
+
def fetch_wiki_pages(wiki_url: str, max_pages: int = 20) -> List[str]:
|
| 44 |
"""Fetch multiple pages from a wiki by following links."""
|
| 45 |
pages_content = []
|
| 46 |
visited = set()
|
| 47 |
+
links_to_visit = []
|
| 48 |
|
| 49 |
try:
|
| 50 |
# Start with the main page
|
|
|
|
| 59 |
lines = (line.strip() for line in text.splitlines())
|
| 60 |
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
| 61 |
text = '\n'.join(chunk for chunk in chunks if chunk)
|
| 62 |
+
if text:
|
| 63 |
+
pages_content.append(text)
|
| 64 |
+
visited.add(wiki_url)
|
| 65 |
|
| 66 |
+
# Collect all links from the main page
|
| 67 |
base_url = '/'.join(wiki_url.split('/')[:3])
|
| 68 |
links = soup.find_all('a', href=True)
|
| 69 |
|
| 70 |
+
for link in links:
|
| 71 |
href = link.get('href')
|
| 72 |
+
if not href:
|
| 73 |
+
continue
|
| 74 |
+
|
| 75 |
+
# Build full URL
|
| 76 |
+
if href.startswith('/'):
|
| 77 |
full_url = base_url + href
|
| 78 |
+
elif href.startswith('http'):
|
| 79 |
+
# Only include links from the same domain
|
| 80 |
+
if base_url in href:
|
| 81 |
+
full_url = href
|
| 82 |
+
else:
|
| 83 |
+
continue
|
| 84 |
else:
|
| 85 |
continue
|
| 86 |
|
| 87 |
+
# Skip if already visited or in queue
|
| 88 |
+
if full_url not in visited and full_url not in links_to_visit:
|
| 89 |
+
# Filter out common non-content links
|
| 90 |
+
skip_patterns = ['#', 'javascript:', 'mailto:', '.pdf', '.jpg', '.png', '.gif',
|
| 91 |
+
'login', 'logout', 'register', 'search', 'edit', 'history']
|
| 92 |
+
if not any(pattern in href.lower() for pattern in skip_patterns):
|
| 93 |
+
links_to_visit.append(full_url)
|
| 94 |
+
|
| 95 |
+
# Continue fetching pages until we reach max_pages or run out of links
|
| 96 |
+
while len(pages_content) < max_pages and links_to_visit:
|
| 97 |
+
current_url = links_to_visit.pop(0)
|
| 98 |
+
|
| 99 |
+
if current_url in visited:
|
| 100 |
+
continue
|
| 101 |
+
|
| 102 |
+
try:
|
| 103 |
+
page_response = requests.get(current_url, timeout=10, allow_redirects=True)
|
| 104 |
+
if page_response.status_code == 200:
|
| 105 |
+
page_soup = BeautifulSoup(page_response.content, 'html.parser')
|
| 106 |
+
for script in page_soup(["script", "style", "nav", "footer", "header"]):
|
| 107 |
+
script.decompose()
|
| 108 |
+
page_text = page_soup.get_text()
|
| 109 |
+
lines = (line.strip() for line in page_text.splitlines())
|
| 110 |
+
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
| 111 |
+
page_text = '\n'.join(chunk for chunk in chunks if chunk)
|
| 112 |
+
|
| 113 |
+
# Only add if there's substantial content
|
| 114 |
+
if page_text and len(page_text.strip()) > 100:
|
| 115 |
+
pages_content.append(page_text)
|
| 116 |
+
visited.add(current_url)
|
| 117 |
+
|
| 118 |
+
# Optionally collect more links from this page if we need more pages
|
| 119 |
+
if len(pages_content) < max_pages:
|
| 120 |
+
page_links = page_soup.find_all('a', href=True)
|
| 121 |
+
for link in page_links:
|
| 122 |
+
href = link.get('href')
|
| 123 |
+
if not href:
|
| 124 |
+
continue
|
| 125 |
+
|
| 126 |
+
if href.startswith('/'):
|
| 127 |
+
new_url = base_url + href
|
| 128 |
+
elif href.startswith('http') and base_url in href:
|
| 129 |
+
new_url = href
|
| 130 |
+
else:
|
| 131 |
+
continue
|
| 132 |
+
|
| 133 |
+
if new_url not in visited and new_url not in links_to_visit:
|
| 134 |
+
skip_patterns = ['#', 'javascript:', 'mailto:', '.pdf', '.jpg',
|
| 135 |
+
'.png', '.gif', 'login', 'logout', 'register',
|
| 136 |
+
'search', 'edit', 'history']
|
| 137 |
+
if not any(pattern in href.lower() for pattern in skip_patterns):
|
| 138 |
+
links_to_visit.append(new_url)
|
| 139 |
+
except Exception as e:
|
| 140 |
+
# Skip failed pages and continue
|
| 141 |
+
continue
|
| 142 |
|
| 143 |
return pages_content
|
| 144 |
except Exception as e:
|
| 145 |
raise Exception(f"Error fetching wiki pages: {str(e)}")
|
| 146 |
|
| 147 |
+
def initialize_rag(wiki_url_input: str, api_key_input: str, max_pages: int = 20, progress=gr.Progress()) -> str:
|
| 148 |
"""Initialize the RAG system with wiki content."""
|
| 149 |
global vector_store, qa_chain, wiki_url, api_key
|
| 150 |
|
|
|
|
| 160 |
os.environ["OPENAI_API_KEY"] = api_key
|
| 161 |
|
| 162 |
progress(0.2, desc="Fetching wiki content...")
|
| 163 |
+
# Fetch wiki pages (convert to int in case it comes as float from Number input)
|
| 164 |
+
documents = fetch_wiki_pages(wiki_url_input, max_pages=int(max_pages))
|
| 165 |
|
| 166 |
if not documents:
|
| 167 |
return "❌ Failed to fetch wiki content. Please check the URL."
|
|
|
|
| 261 |
value=""
|
| 262 |
)
|
| 263 |
with gr.Column(scale=1):
|
| 264 |
+
max_pages_input = gr.Number(
|
| 265 |
+
label="Max Pages",
|
| 266 |
+
value=20,
|
| 267 |
+
minimum=1,
|
| 268 |
+
maximum=100,
|
| 269 |
+
step=1,
|
| 270 |
+
info="Number of pages to fetch"
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
with gr.Row():
|
| 274 |
+
init_btn = gr.Button("Initialize RAG", variant="primary", scale=1)
|
| 275 |
|
| 276 |
status_output = gr.Textbox(
|
| 277 |
label="Status",
|
|
|
|
| 299 |
# Event handlers
|
| 300 |
init_btn.click(
|
| 301 |
fn=initialize_rag,
|
| 302 |
+
inputs=[wiki_url_input, api_key_input, max_pages_input],
|
| 303 |
outputs=status_output
|
| 304 |
)
|
| 305 |
|