OrganizingForChange commited on
Commit
75b69ea
·
verified ·
1 Parent(s): eab4de1

Upload 8 files

Browse files
Files changed (1) hide show
  1. app.py +88 -29
app.py CHANGED
@@ -40,10 +40,11 @@ def fetch_wiki_content(wiki_url: str) -> List[str]:
40
  except Exception as e:
41
  raise Exception(f"Error fetching wiki content: {str(e)}")
42
 
43
- def fetch_wiki_pages(wiki_url: str, max_pages: int = 10) -> List[str]:
44
  """Fetch multiple pages from a wiki by following links."""
45
  pages_content = []
46
  visited = set()
 
47
 
48
  try:
49
  # Start with the main page
@@ -58,44 +59,92 @@ def fetch_wiki_pages(wiki_url: str, max_pages: int = 10) -> List[str]:
58
  lines = (line.strip() for line in text.splitlines())
59
  chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
60
  text = '\n'.join(chunk for chunk in chunks if chunk)
61
- pages_content.append(text)
62
- visited.add(wiki_url)
 
63
 
64
- # Try to find and follow wiki links
65
  base_url = '/'.join(wiki_url.split('/')[:3])
66
  links = soup.find_all('a', href=True)
67
 
68
- for link in links[:max_pages-1]:
69
  href = link.get('href')
70
- if href and href.startswith('/'):
 
 
 
 
71
  full_url = base_url + href
72
- elif href and href.startswith('http'):
73
- full_url = href
 
 
 
 
74
  else:
75
  continue
76
 
77
- if full_url not in visited and len(pages_content) < max_pages:
78
- try:
79
- page_response = requests.get(full_url, timeout=5)
80
- if page_response.status_code == 200:
81
- page_soup = BeautifulSoup(page_response.content, 'html.parser')
82
- for script in page_soup(["script", "style"]):
83
- script.decompose()
84
- page_text = page_soup.get_text()
85
- lines = (line.strip() for line in page_text.splitlines())
86
- chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
87
- page_text = '\n'.join(chunk for chunk in chunks if chunk)
88
- if page_text:
89
- pages_content.append(page_text)
90
- visited.add(full_url)
91
- except:
92
- continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
 
94
  return pages_content
95
  except Exception as e:
96
  raise Exception(f"Error fetching wiki pages: {str(e)}")
97
 
98
- def initialize_rag(wiki_url_input: str, api_key_input: str, progress=gr.Progress()) -> str:
99
  """Initialize the RAG system with wiki content."""
100
  global vector_store, qa_chain, wiki_url, api_key
101
 
@@ -111,8 +160,8 @@ def initialize_rag(wiki_url_input: str, api_key_input: str, progress=gr.Progress
111
  os.environ["OPENAI_API_KEY"] = api_key
112
 
113
  progress(0.2, desc="Fetching wiki content...")
114
- # Fetch wiki pages
115
- documents = fetch_wiki_pages(wiki_url_input, max_pages=10)
116
 
117
  if not documents:
118
  return "❌ Failed to fetch wiki content. Please check the URL."
@@ -212,7 +261,17 @@ with gr.Blocks(title="Chat with Wiki Documents") as app:
212
  value=""
213
  )
214
  with gr.Column(scale=1):
215
- init_btn = gr.Button("Initialize RAG", variant="primary")
 
 
 
 
 
 
 
 
 
 
216
 
217
  status_output = gr.Textbox(
218
  label="Status",
@@ -240,7 +299,7 @@ with gr.Blocks(title="Chat with Wiki Documents") as app:
240
  # Event handlers
241
  init_btn.click(
242
  fn=initialize_rag,
243
- inputs=[wiki_url_input, api_key_input],
244
  outputs=status_output
245
  )
246
 
 
40
  except Exception as e:
41
  raise Exception(f"Error fetching wiki content: {str(e)}")
42
 
43
+ def fetch_wiki_pages(wiki_url: str, max_pages: int = 20) -> List[str]:
44
  """Fetch multiple pages from a wiki by following links."""
45
  pages_content = []
46
  visited = set()
47
+ links_to_visit = []
48
 
49
  try:
50
  # Start with the main page
 
59
  lines = (line.strip() for line in text.splitlines())
60
  chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
61
  text = '\n'.join(chunk for chunk in chunks if chunk)
62
+ if text:
63
+ pages_content.append(text)
64
+ visited.add(wiki_url)
65
 
66
+ # Collect all links from the main page
67
  base_url = '/'.join(wiki_url.split('/')[:3])
68
  links = soup.find_all('a', href=True)
69
 
70
+ for link in links:
71
  href = link.get('href')
72
+ if not href:
73
+ continue
74
+
75
+ # Build full URL
76
+ if href.startswith('/'):
77
  full_url = base_url + href
78
+ elif href.startswith('http'):
79
+ # Only include links from the same domain
80
+ if base_url in href:
81
+ full_url = href
82
+ else:
83
+ continue
84
  else:
85
  continue
86
 
87
+ # Skip if already visited or in queue
88
+ if full_url not in visited and full_url not in links_to_visit:
89
+ # Filter out common non-content links
90
+ skip_patterns = ['#', 'javascript:', 'mailto:', '.pdf', '.jpg', '.png', '.gif',
91
+ 'login', 'logout', 'register', 'search', 'edit', 'history']
92
+ if not any(pattern in href.lower() for pattern in skip_patterns):
93
+ links_to_visit.append(full_url)
94
+
95
+ # Continue fetching pages until we reach max_pages or run out of links
96
+ while len(pages_content) < max_pages and links_to_visit:
97
+ current_url = links_to_visit.pop(0)
98
+
99
+ if current_url in visited:
100
+ continue
101
+
102
+ try:
103
+ page_response = requests.get(current_url, timeout=10, allow_redirects=True)
104
+ if page_response.status_code == 200:
105
+ page_soup = BeautifulSoup(page_response.content, 'html.parser')
106
+ for script in page_soup(["script", "style", "nav", "footer", "header"]):
107
+ script.decompose()
108
+ page_text = page_soup.get_text()
109
+ lines = (line.strip() for line in page_text.splitlines())
110
+ chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
111
+ page_text = '\n'.join(chunk for chunk in chunks if chunk)
112
+
113
+ # Only add if there's substantial content
114
+ if page_text and len(page_text.strip()) > 100:
115
+ pages_content.append(page_text)
116
+ visited.add(current_url)
117
+
118
+ # Optionally collect more links from this page if we need more pages
119
+ if len(pages_content) < max_pages:
120
+ page_links = page_soup.find_all('a', href=True)
121
+ for link in page_links:
122
+ href = link.get('href')
123
+ if not href:
124
+ continue
125
+
126
+ if href.startswith('/'):
127
+ new_url = base_url + href
128
+ elif href.startswith('http') and base_url in href:
129
+ new_url = href
130
+ else:
131
+ continue
132
+
133
+ if new_url not in visited and new_url not in links_to_visit:
134
+ skip_patterns = ['#', 'javascript:', 'mailto:', '.pdf', '.jpg',
135
+ '.png', '.gif', 'login', 'logout', 'register',
136
+ 'search', 'edit', 'history']
137
+ if not any(pattern in href.lower() for pattern in skip_patterns):
138
+ links_to_visit.append(new_url)
139
+ except Exception as e:
140
+ # Skip failed pages and continue
141
+ continue
142
 
143
  return pages_content
144
  except Exception as e:
145
  raise Exception(f"Error fetching wiki pages: {str(e)}")
146
 
147
+ def initialize_rag(wiki_url_input: str, api_key_input: str, max_pages: int = 20, progress=gr.Progress()) -> str:
148
  """Initialize the RAG system with wiki content."""
149
  global vector_store, qa_chain, wiki_url, api_key
150
 
 
160
  os.environ["OPENAI_API_KEY"] = api_key
161
 
162
  progress(0.2, desc="Fetching wiki content...")
163
+ # Fetch wiki pages (convert to int in case it comes as float from Number input)
164
+ documents = fetch_wiki_pages(wiki_url_input, max_pages=int(max_pages))
165
 
166
  if not documents:
167
  return "❌ Failed to fetch wiki content. Please check the URL."
 
261
  value=""
262
  )
263
  with gr.Column(scale=1):
264
+ max_pages_input = gr.Number(
265
+ label="Max Pages",
266
+ value=20,
267
+ minimum=1,
268
+ maximum=100,
269
+ step=1,
270
+ info="Number of pages to fetch"
271
+ )
272
+
273
+ with gr.Row():
274
+ init_btn = gr.Button("Initialize RAG", variant="primary", scale=1)
275
 
276
  status_output = gr.Textbox(
277
  label="Status",
 
299
  # Event handlers
300
  init_btn.click(
301
  fn=initialize_rag,
302
+ inputs=[wiki_url_input, api_key_input, max_pages_input],
303
  outputs=status_output
304
  )
305