Krixter commited on
Commit
c934003
·
verified ·
1 Parent(s): 0f5842f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +233 -208
app.py CHANGED
@@ -1,220 +1,245 @@
1
- import os
2
- import re
3
- import time
4
  import requests
5
  from bs4 import BeautifulSoup
6
  from urllib.parse import urljoin, urlparse
7
- import gradio as gr
 
 
8
 
9
- # Comprehensive list of headers to look like a real browser
10
  HEADERS = {
11
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
12
- 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
13
- 'Accept-Language': 'en-US,en;q=0.5',
14
- 'Connection': 'keep-alive',
15
- 'Upgrade-Insecure-Requests': '1',
16
  }
17
 
18
- class UniversalNovelScraper:
19
- def __init__(self, delay=1.0):
20
- self.session = requests.Session()
21
- self.session.headers.update(HEADERS)
22
- self.delay = delay
23
-
24
- def clean_text(self, text):
25
- """Cleans up messy text patterns, whitespace, and artifacts."""
26
- if not text:
27
- return ""
28
- # Remove consecutive blank lines
29
- text = re.sub(r'\n\s*\n', '\n\n', text)
30
- return text.strip()
31
-
32
- def extract_title(self, soup):
33
- """Heuristic approach to find the chapter title."""
34
- # Check standard heading tags first
35
- for tag in ['h1', 'h2', 'h3']:
36
- heading = soup.find(tag)
37
- if heading and len(heading.get_text().strip()) > 3:
38
- return heading.get_text().strip()
39
-
40
- # Fallback to page title header minus site branding
41
- page_title = soup.find('title')
42
- if page_title:
43
- title_text = page_title.get_text()
44
- for delimiter in ['|', '-', '—', '::']:
45
- if delimiter in title_text:
46
- title_text = title_text.split(delimiter)[0]
47
- return title_text.strip()
48
-
49
- return "Untitled Chapter"
50
-
51
- def extract_content(self, soup):
52
- """Intelligently pinpoints the main text content block using density analysis."""
53
- best_element = None
54
- max_score = 0
55
-
56
- # Target containers likely to hold story content
57
- for tag in ['div', 'article', 'section']:
58
- for el in soup.find_all(tag):
59
- # Ignore structural headers/footers/nav bars
60
- el_id = str(el.get('id', '')).lower()
61
- el_class = " ".join(el.get('class', [])).lower()
62
- if any(k in el_id or k in el_class for k in ['nav', 'footer', 'comment', 'sidebar', 'menu', 'author-note']):
63
- continue
64
-
65
- # Score based on count of paragraph tags and density of text
66
- p_tags = el.find_all('p', recursive=False)
67
- if not p_tags:
68
- # Fallback to nested paragraphs if flat structure isn't found
69
- p_tags = el.find_all('p')
70
-
71
- score = len(p_tags)
72
- if score > max_score:
73
- max_score = score
74
- best_element = el
75
-
76
- if best_element:
77
- # Gather paragraph text, weeding out empty lines and short social share snippets
78
- paragraphs = []
79
- for p in best_element.find_all('p'):
80
- p_text = p.get_text().strip()
81
- if len(p_text) > 10 and not any(x in p_text.lower() for x in ['share this', 'subscribe', 'patreon', 'discord']):
82
- paragraphs.append(p_text)
83
- return "\n\n".join(paragraphs)
84
-
85
- # Fallback absolute layout match (text element grouping)
86
- paragraphs = [p.get_text().strip() for p in soup.find_all('p') if len(p.get_text().strip()) > 20]
87
- return "\n\n".join(paragraphs) if paragraphs else None
88
-
89
- def find_next_url(self, soup, current_url):
90
- """Finds the next chapter link using deep regex matching on text and anchor metadata."""
91
- next_patterns = [
92
- re.compile(r'\bnext\b', re.I),
93
- re.compile(r'next\s*chapter', re.I),
94
- re.compile(r'forward', re.I),
95
- re.compile(r'›'),
96
- re.compile(r'»'),
97
- re.compile(r'→')
98
- ]
99
-
100
- for a in soup.find_all('a', href=True):
101
- text = a.get_text().strip()
102
- href = a['href']
103
-
104
- # 1. Match based on visible hyperlink text
105
- for pattern in next_patterns:
106
- if pattern.search(text):
107
- return urljoin(current_url, href)
108
-
109
- # 2. Match based on structural elements (CSS classes, IDs)
110
- for attr in ['class', 'id', 'rel']:
111
- val = a.get(attr)
112
- if val:
113
- val_str = " ".join(val) if isinstance(val, list) else str(val)
114
- if 'next' in val_str.lower() and not 'prev' in val_str.lower():
115
- return urljoin(current_url, href)
116
-
117
- return None
118
-
119
- def scrape_novel(self, start_url, max_chapters=50):
120
- current_url = start_url
121
- chapters_scraped = 0
122
- novel_output = []
123
- log_history = []
124
-
125
- log_history.append(f"🚀 Starting Engine for: {start_url}")
126
- yield "\n".join(log_history), None
127
-
128
- while current_url and chapters_scraped < max_chapters:
129
- try:
130
- time.sleep(self.delay) # Graceful request throttling
131
- response = self.session.get(current_url, timeout=15)
132
-
133
- if response.status_code != 200:
134
- log_history.append(f"❌ Failed to fetch (HTTP {response.status_code}): {current_url}")
135
- break
136
-
137
- soup = BeautifulSoup(response.content, 'lxml')
138
-
139
- title = self.extract_title(soup)
140
- content = self.extract_content(soup)
141
-
142
- if not content or len(content) < 200:
143
- log_history.append(f"⚠️ Warning: Found minimal content at chapter {chapters_scraped+1}. Skipping blank content.")
144
- else:
145
- chapters_scraped += 1
146
- formatted_chapter = f"=== {title} ===\n\n{self.clean_text(content)}\n\n\n"
147
- novel_output.append(formatted_chapter)
148
- log_history.append(f"✅ Scraped ({chapters_scraped}/{max_chapters}): {title}")
149
-
150
- # Look for the next sequence link
151
- next_url = self.find_next_url(soup, current_url)
152
-
153
- # Prevent infinite loops on identical URLs
154
- if next_url == current_url:
155
- log_history.append("🛑 'Next' link loops back to current page. Halting.")
156
- break
157
-
158
- current_url = next_url
159
- yield "\n".join(log_history), None
160
-
161
- except Exception as e:
162
- log_history.append(f"💥 Error encountered: {str(e)}")
163
- break
164
-
165
- if novel_output:
166
- # Generate a valid filename safe for environments
167
- parsed_url = urlparse(start_url)
168
- filename = f"scraped_novel_{parsed_url.netloc.replace('.', '_')}.txt"
169
-
170
- with open(filename, "w", encoding="utf-8") as f:
171
- f.write("".join(novel_output))
172
-
173
- log_history.append(f"\n🎉 Extraction Complete! Saved {chapters_scraped} chapters successfully.")
174
- yield "\n".join(log_history), filename
175
- else:
176
- log_history.append("\n❌ Failed to extract any valid content.")
177
- yield "\n".join(log_history), None
178
-
179
- # --- Gradio UI Engine Setup ---
180
- def run_scraper(url, max_chapters, delay):
181
- if not url.startswith(("http://", "https://")):
182
- return "❌ Please enter a valid HTTP/HTTPS URL.", None
183
-
184
- scraper = UniversalNovelScraper(delay=delay)
185
- # Using generator pattern to feed real-time outputs to UI without timing out
186
- for log, file_path in scraper.scrape_novel(url, int(max_chapters)):
187
- yield log, file_path
188
-
189
- with gr.Blocks(theme=gr.themes.Soft()) as demo:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
  gr.Markdown(
191
  """
192
- # 📚 Universal Webnovel Scraper
193
- Paste the link to the **first chapter** of any webnovel site. The parser will read the text and attempt to automatically crawl sequential chapters via "Next" buttons.
194
- """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
195
  )
196
-
197
- with gr.Row():
198
- with gr.Column(scale=2):
199
- url_input = gr.Textbox(
200
- label="First Chapter URL",
201
- placeholder="https://www.royalroad.com/fiction/.../chapter-1"
202
- )
203
- with gr.Row():
204
- max_chaps = gr.Number(label="Max Chapters Limit", value=20, precision=0)
205
- delay_input = gr.Slider(label="Politeness Delay (seconds)", minimum=0.5, maximum=3.0, value=1.0, step=0.5)
206
-
207
- submit_btn = gr.Button("⚡ Start Scraping", variant="primary")
208
-
209
- with gr.Column(scale=3):
210
- status_logs = gr.Textbox(label="Real-time Engine Logs", lines=12, max_lines=15, interactive=False)
211
- file_output = gr.File(label="Download Generated .TXT File")
212
-
213
- submit_btn.click(
214
- fn=run_scraper,
215
- inputs=[url_input, max_chaps, delay_input],
216
- outputs=[status_logs, file_output]
217
  )
218
 
219
- if __name__ == "__main__":
220
- demo.queue().launch()
 
1
+ import gradio as gr
 
 
2
  import requests
3
  from bs4 import BeautifulSoup
4
  from urllib.parse import urljoin, urlparse
5
+ import time
6
+ import re
7
+ import os
8
 
 
9
  HEADERS = {
10
+ "User-Agent": (
11
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
12
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
13
+ "Chrome/124.0 Safari/537.36"
14
+ )
15
  }
16
 
17
+ SESSION = requests.Session()
18
+ SESSION.headers.update(HEADERS)
19
+
20
+
21
+ # ---------- HELPERS ----------
22
+
23
+ def clean_text(text):
24
+ text = re.sub(r"\n{3,}", "\n\n", text)
25
+ text = re.sub(r"[ \t]+", " ", text)
26
+ return text.strip()
27
+
28
+
29
+ def is_chapter_link(text, href):
30
+ if not href:
31
+ return False
32
+ t = (text or "").lower()
33
+ h = href.lower()
34
+
35
+ keywords = [
36
+ "chapter",
37
+ "chap",
38
+ "episode",
39
+ "ep",
40
+ ]
41
+
42
+ return any(k in t or k in h for k in keywords)
43
+
44
+
45
+ def fetch(url):
46
+ r = SESSION.get(url, timeout=20)
47
+ r.raise_for_status()
48
+ return r.text
49
+
50
+
51
+ def extract_title(soup):
52
+ if soup.title:
53
+ return soup.title.get_text(strip=True)
54
+
55
+ h1 = soup.find("h1")
56
+ if h1:
57
+ return h1.get_text(strip=True)
58
+
59
+ return "Novel"
60
+
61
+
62
+ def extract_chapter_content(soup):
63
+ selectors = [
64
+ ".chapter-content",
65
+ ".entry-content",
66
+ ".text-left",
67
+ ".chapter_body",
68
+ ".reading-content",
69
+ "#chapter-content",
70
+ "#content",
71
+ ".content",
72
+ "article",
73
+ ]
74
+
75
+ for selector in selectors:
76
+ node = soup.select_one(selector)
77
+ if node:
78
+ for bad in node(["script", "style", "iframe", "ins"]):
79
+ bad.decompose()
80
+
81
+ txt = node.get_text("\n", strip=True)
82
+ if len(txt) > 300:
83
+ return txt
84
+
85
+ body = soup.get_text("\n", strip=True)
86
+ return body
87
+
88
+
89
+ def find_next_chapter(soup, current_url):
90
+ candidates = []
91
+
92
+ for a in soup.find_all("a", href=True):
93
+ txt = a.get_text(" ", strip=True).lower()
94
+ href = urljoin(current_url, a["href"])
95
+
96
+ if any(
97
+ k in txt
98
+ for k in ["next", "next chapter", "continue", "→", ">>"]
99
+ ):
100
+ candidates.append(href)
101
+
102
+ return candidates[0] if candidates else None
103
+
104
+
105
+ def collect_toc_links(soup, base_url):
106
+ seen = set()
107
+ links = []
108
+
109
+ for a in soup.find_all("a", href=True):
110
+ txt = a.get_text(" ", strip=True)
111
+ href = urljoin(base_url, a["href"])
112
+
113
+ if is_chapter_link(txt, href):
114
+ if href not in seen:
115
+ seen.add(href)
116
+ links.append(href)
117
+
118
+ return links
119
+
120
+
121
+ # ---------- MAIN ----------
122
+
123
+ def scrape_novel(url, max_chapters, delay_sec, filename):
124
+ logs = []
125
+
126
+ def log(msg):
127
+ logs.append(msg)
128
+
129
+ try:
130
+ html = fetch(url)
131
+ except Exception as e:
132
+ return "\n".join(logs + [f"Failed: {e}"]), None
133
+
134
+ soup = BeautifulSoup(html, "html.parser")
135
+
136
+ title = extract_title(soup)
137
+
138
+ if not filename.strip():
139
+ filename = re.sub(r"[^\w\- ]+", "", title).strip() or "novel"
140
+
141
+ all_text = []
142
+
143
+ chapter_links = collect_toc_links(soup, url)
144
+
145
+ # If no TOC found → treat URL as chapter
146
+ if not chapter_links:
147
+ chapter_links = [url]
148
+
149
+ visited = set()
150
+ count = 0
151
+
152
+ i = 0
153
+
154
+ while i < len(chapter_links):
155
+ if max_chapters and count >= max_chapters:
156
+ break
157
+
158
+ chapter_url = chapter_links[i]
159
+
160
+ if chapter_url in visited:
161
+ i += 1
162
+ continue
163
+
164
+ visited.add(chapter_url)
165
+
166
+ try:
167
+ page = fetch(chapter_url)
168
+ s = BeautifulSoup(page, "html.parser")
169
+
170
+ chap_title = extract_title(s)
171
+ chap_body = extract_chapter_content(s)
172
+
173
+ all_text.append(f"\n\n{chap_title}\n")
174
+ all_text.append(chap_body)
175
+
176
+ log(f"✓ {chap_title}")
177
+ count += 1
178
+
179
+ nxt = find_next_chapter(s, chapter_url)
180
+
181
+ if nxt and nxt not in visited:
182
+ chapter_links.append(nxt)
183
+
184
+ time.sleep(delay_sec)
185
+
186
+ except Exception as e:
187
+ log(f"✗ {chapter_url} -> {e}")
188
+
189
+ i += 1
190
+
191
+ final_text = clean_text("\n".join(all_text))
192
+
193
+ path = f"{filename}.txt"
194
+
195
+ with open(path, "w", encoding="utf-8") as f:
196
+ f.write(final_text)
197
+
198
+ log(f"\nDone. Chapters: {count}")
199
+
200
+ return "\n".join(logs), path
201
+
202
+
203
+ # ---------- UI ----------
204
+
205
+ with gr.Blocks(title="Universal Webnovel Scraper") as demo:
206
  gr.Markdown(
207
  """
208
+ # Universal Webnovel Scraper
209
+
210
+ Paste a webnovel link.
211
+ Supports TOC pages + direct chapters.
212
+ Downloads clean TXT.
213
+ """
214
+ )
215
+
216
+ url = gr.Textbox(label="Novel URL")
217
+
218
+ max_chapters = gr.Number(
219
+ label="Max Chapters (0 = unlimited)",
220
+ value=50,
221
+ precision=0,
222
+ )
223
+
224
+ delay_sec = gr.Slider(
225
+ 0,
226
+ 5,
227
+ value=1,
228
+ step=0.5,
229
+ label="Delay Between Requests",
230
  )
231
+
232
+ filename = gr.Textbox(label="Filename (optional)")
233
+
234
+ btn = gr.Button("Scrape")
235
+
236
+ logs = gr.Textbox(lines=18, label="Logs")
237
+ file_out = gr.File(label="Download TXT")
238
+
239
+ btn.click(
240
+ scrape_novel,
241
+ inputs=[url, max_chapters, delay_sec, filename],
242
+ outputs=[logs, file_out],
 
 
 
 
 
 
 
 
 
243
  )
244
 
245
+ demo.launch()