Krixter commited on
Commit
742602f
Β·
verified Β·
1 Parent(s): c934003

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +622 -211
app.py CHANGED
@@ -1,245 +1,656 @@
1
  import gradio as gr
2
  import requests
3
  from bs4 import BeautifulSoup
4
- from urllib.parse import urljoin, urlparse
5
- import time
6
  import re
 
7
  import os
8
-
 
 
 
 
 
 
 
9
  HEADERS = {
10
  "User-Agent": (
11
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
12
  "AppleWebKit/537.36 (KHTML, like Gecko) "
13
- "Chrome/124.0 Safari/537.36"
14
- )
 
 
15
  }
16
 
17
  SESSION = requests.Session()
18
  SESSION.headers.update(HEADERS)
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
- # ---------- HELPERS ----------
22
-
23
- def clean_text(text):
24
- text = re.sub(r"\n{3,}", "\n\n", text)
25
- text = re.sub(r"[ \t]+", " ", text)
26
- return text.strip()
27
-
28
-
29
- def is_chapter_link(text, href):
30
- if not href:
31
- return False
32
- t = (text or "").lower()
33
- h = href.lower()
34
-
35
- keywords = [
36
- "chapter",
37
- "chap",
38
- "episode",
39
- "ep",
40
- ]
41
-
42
- return any(k in t or k in h for k in keywords)
43
-
44
-
45
- def fetch(url):
46
- r = SESSION.get(url, timeout=20)
47
- r.raise_for_status()
48
- return r.text
49
-
50
-
51
- def extract_title(soup):
52
- if soup.title:
53
- return soup.title.get_text(strip=True)
54
-
55
- h1 = soup.find("h1")
56
- if h1:
57
- return h1.get_text(strip=True)
58
-
59
- return "Novel"
60
-
61
-
62
- def extract_chapter_content(soup):
63
- selectors = [
64
- ".chapter-content",
65
- ".entry-content",
66
- ".text-left",
67
- ".chapter_body",
68
- ".reading-content",
69
- "#chapter-content",
70
- "#content",
71
- ".content",
72
- "article",
73
- ]
74
-
75
- for selector in selectors:
76
- node = soup.select_one(selector)
77
- if node:
78
- for bad in node(["script", "style", "iframe", "ins"]):
79
- bad.decompose()
80
-
81
- txt = node.get_text("\n", strip=True)
82
- if len(txt) > 300:
83
- return txt
84
-
85
- body = soup.get_text("\n", strip=True)
86
- return body
87
-
88
-
89
- def find_next_chapter(soup, current_url):
90
- candidates = []
91
-
92
- for a in soup.find_all("a", href=True):
93
- txt = a.get_text(" ", strip=True).lower()
94
- href = urljoin(current_url, a["href"])
95
-
96
- if any(
97
- k in txt
98
- for k in ["next", "next chapter", "continue", "β†’", ">>"]
99
- ):
100
- candidates.append(href)
101
-
102
- return candidates[0] if candidates else None
103
-
104
-
105
- def collect_toc_links(soup, base_url):
106
- seen = set()
 
107
  links = []
108
 
109
- for a in soup.find_all("a", href=True):
110
- txt = a.get_text(" ", strip=True)
111
- href = urljoin(base_url, a["href"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112
 
113
- if is_chapter_link(txt, href):
114
- if href not in seen:
115
- seen.add(href)
116
- links.append(href)
 
 
 
 
 
117
 
 
118
  return links
119
 
120
 
121
- # ---------- MAIN ----------
122
-
123
- def scrape_novel(url, max_chapters, delay_sec, filename):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  logs = []
 
125
 
126
  def log(msg):
127
  logs.append(msg)
128
-
129
- try:
130
- html = fetch(url)
131
- except Exception as e:
132
- return "\n".join(logs + [f"Failed: {e}"]), None
133
-
134
- soup = BeautifulSoup(html, "html.parser")
135
-
136
- title = extract_title(soup)
137
-
138
- if not filename.strip():
139
- filename = re.sub(r"[^\w\- ]+", "", title).strip() or "novel"
140
-
141
- all_text = []
142
-
143
- chapter_links = collect_toc_links(soup, url)
144
-
145
- # If no TOC found β†’ treat URL as chapter
146
- if not chapter_links:
147
- chapter_links = [url]
148
-
149
- visited = set()
150
- count = 0
151
-
152
- i = 0
153
-
154
- while i < len(chapter_links):
155
- if max_chapters and count >= max_chapters:
156
- break
157
-
158
- chapter_url = chapter_links[i]
159
-
160
- if chapter_url in visited:
161
- i += 1
162
- continue
163
-
164
- visited.add(chapter_url)
165
-
166
- try:
167
- page = fetch(chapter_url)
168
- s = BeautifulSoup(page, "html.parser")
169
-
170
- chap_title = extract_title(s)
171
- chap_body = extract_chapter_content(s)
172
-
173
- all_text.append(f"\n\n{chap_title}\n")
174
- all_text.append(chap_body)
175
-
176
- log(f"βœ“ {chap_title}")
177
- count += 1
178
-
179
- nxt = find_next_chapter(s, chapter_url)
180
-
181
- if nxt and nxt not in visited:
182
- chapter_links.append(nxt)
183
-
184
- time.sleep(delay_sec)
185
-
186
- except Exception as e:
187
- log(f"βœ— {chapter_url} -> {e}")
188
-
189
- i += 1
190
-
191
- final_text = clean_text("\n".join(all_text))
192
-
193
- path = f"{filename}.txt"
194
-
195
- with open(path, "w", encoding="utf-8") as f:
196
- f.write(final_text)
197
-
198
- log(f"\nDone. Chapters: {count}")
199
-
200
- return "\n".join(logs), path
201
-
202
-
203
- # ---------- UI ----------
204
-
205
- with gr.Blocks(title="Universal Webnovel Scraper") as demo:
206
- gr.Markdown(
207
- """
208
- # Universal Webnovel Scraper
209
-
210
- Paste a webnovel link.
211
- Supports TOC pages + direct chapters.
212
- Downloads clean TXT.
213
- """
214
  )
215
 
216
- url = gr.Textbox(label="Novel URL")
217
-
218
- max_chapters = gr.Number(
219
- label="Max Chapters (0 = unlimited)",
220
- value=50,
221
- precision=0,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
 
224
- delay_sec = gr.Slider(
225
- 0,
226
- 5,
227
- value=1,
228
- step=0.5,
229
- label="Delay Between Requests",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
230
  )
231
 
232
- filename = gr.Textbox(label="Filename (optional)")
233
-
234
- btn = gr.Button("Scrape")
235
-
236
- logs = gr.Textbox(lines=18, label="Logs")
237
- file_out = gr.File(label="Download TXT")
238
-
239
- btn.click(
240
- scrape_novel,
241
- inputs=[url, max_chapters, delay_sec, filename],
242
- outputs=[logs, file_out],
243
  )
244
 
245
- demo.launch()
 
 
1
  import gradio as gr
2
  import requests
3
  from bs4 import BeautifulSoup
 
 
4
  import re
5
+ import time
6
  import os
7
+ import tempfile
8
+ import json
9
+ from urllib.parse import urlparse, urljoin
10
+ import urllib.robotparser
11
+
12
+ # ─────────────────────────────────────────────
13
+ # HEADERS & SESSION
14
+ # ─────────────────────────────────────────────
15
  HEADERS = {
16
  "User-Agent": (
17
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
18
  "AppleWebKit/537.36 (KHTML, like Gecko) "
19
+ "Chrome/124.0.0.0 Safari/537.36"
20
+ ),
21
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
22
+ "Accept-Language": "en-US,en;q=0.5",
23
  }
24
 
25
  SESSION = requests.Session()
26
  SESSION.headers.update(HEADERS)
27
 
28
+ # ─────────────────────────────────────────────
29
+ # SITE PROFILES
30
+ # Each profile tells the scraper HOW to parse
31
+ # a specific site's HTML structure.
32
+ # ─────────────────────────────────────────────
33
+ SITE_PROFILES = {
34
+ # ── RoyalRoad ────────────────────────────
35
+ "royalroad.com": {
36
+ "name": "Royal Road",
37
+ "novel_title": ("h1", {"class": "font-white"}),
38
+ "novel_author": ("span", {"property": "name"}),
39
+ "chapter_list": ("table", {"id": "chapters"}),
40
+ "chapter_links": ("a", {"href": re.compile(r"/fiction/\d+/chapter/")}),
41
+ "chapter_title": ("h1", {"class": "font-white"}),
42
+ "content_sel": ("div", {"class": "chapter-content"}),
43
+ "next_page": None,
44
+ "delay": 1.2,
45
+ },
46
+ # ── ScribbleHub ──────────────────────────
47
+ "scribblehub.com": {
48
+ "name": "ScribbleHub",
49
+ "novel_title": ("div", {"class": "fic_title"}),
50
+ "novel_author": ("span", {"class": "auth_name_fic"}),
51
+ "chapter_list": None, # uses API
52
+ "api_toc": "https://www.scribblehub.com/wp-admin/admin-ajax.php",
53
+ "chapter_links": ("a", {"class": "toc_a"}),
54
+ "chapter_title": ("div", {"class": "chapter-title"}),
55
+ "content_sel": ("div", {"class": "chp_raw"}),
56
+ "next_page": ("a", {"class": "btn-chapter-next"}),
57
+ "delay": 1.5,
58
+ },
59
+ # ── WuxiaWorld ───────────────────────────
60
+ "wuxiaworld.com": {
61
+ "name": "WuxiaWorld",
62
+ "novel_title": ("h3", {"class": "novel-title"}),
63
+ "novel_author": ("span", {"class": "author"}),
64
+ "chapter_list": ("div", {"class": "chapter-list"}),
65
+ "chapter_links": ("a", {"href": re.compile(r"/novel/[^/]+/[^/]+-chapter")}),
66
+ "chapter_title": ("h4", {}),
67
+ "content_sel": ("div", {"class": "chapter-content"}),
68
+ "next_page": None,
69
+ "delay": 1.5,
70
+ },
71
+ # ── NovelFull / NovelFull.com ─────────────
72
+ "novelfull.com": {
73
+ "name": "NovelFull",
74
+ "novel_title": ("h3", {"class": "title"}),
75
+ "novel_author": ("a", {"itemprop": "author"}),
76
+ "chapter_list": ("ul", {"class": "list-chapter"}),
77
+ "chapter_links": ("a", {"href": re.compile(r"\.html")}),
78
+ "chapter_title": ("a", {"class": "chr-title"}),
79
+ "content_sel": ("div", {"id": "chapter-content"}),
80
+ "next_page": ("a", {"id": "next_chap"}),
81
+ "delay": 1.0,
82
+ },
83
+ # ── LightNovelPub ────────────────────────
84
+ "lightnovelpub.com": {
85
+ "name": "LightNovelPub",
86
+ "novel_title": ("h1", {"class": "novel-title"}),
87
+ "novel_author": ("span", {"itemprop": "author"}),
88
+ "chapter_list": ("ul", {"class": "chapter-list"}),
89
+ "chapter_links": ("a", {"href": re.compile(r"/novel/.+/chapter-\d+")}),
90
+ "chapter_title": ("span", {"class": "chapter-title"}),
91
+ "content_sel": ("div", {"class": "chapter-content"}),
92
+ "next_page": ("a", {"class": "btn-next"}),
93
+ "delay": 1.2,
94
+ },
95
+ # ── FreeWebNovel ─────────────────────────
96
+ "freewebnovel.com": {
97
+ "name": "FreeWebNovel",
98
+ "novel_title": ("h1", {"class": "pc_title"}),
99
+ "novel_author": ("a", {"class": "a1"}),
100
+ "chapter_list": ("ul", {"class": "ul-list5"}),
101
+ "chapter_links": ("a", {"href": re.compile(r"\.html")}),
102
+ "chapter_title": ("h1", {"class": "chapter-title"}),
103
+ "content_sel": ("div", {"class": "txt"}),
104
+ "next_page": ("a", {"id": "idnext"}),
105
+ "delay": 1.0,
106
+ },
107
+ # ── NovelUpdates (info page, links out) ──
108
+ "novelupdates.com": {
109
+ "name": "NovelUpdates",
110
+ "novel_title": ("div", {"class": "seriestitlenu"}),
111
+ "novel_author": ("div", {"id": "authtag"}),
112
+ "chapter_list": ("table", {"id": "myTable"}),
113
+ "chapter_links": ("a", {"class": "chp-release"}),
114
+ "chapter_title": None,
115
+ "content_sel": None,
116
+ "next_page": None,
117
+ "delay": 1.5,
118
+ "_external_links": True, # chapters hosted elsewhere
119
+ },
120
+ # ── WebNovel.com ─────────────────────────
121
+ "webnovel.com": {
122
+ "name": "WebNovel",
123
+ "novel_title": ("h2", {"class": "dib"}),
124
+ "novel_author": ("address", {}),
125
+ "chapter_list": ("ol", {"class": "c_list"}),
126
+ "chapter_links": ("a", {"href": re.compile(r"/book/\d+/\d+")}),
127
+ "chapter_title": ("h3", {"class": "cha-tit"}),
128
+ "content_sel": ("div", {"class": "cha-words"}),
129
+ "next_page": ("a", {"class": "_nextChapter"}),
130
+ "delay": 2.0,
131
+ },
132
+ # ── Wattpad ──────────────────────────────
133
+ "wattpad.com": {
134
+ "name": "Wattpad",
135
+ "novel_title": ("h1", {"class": "story-title"}),
136
+ "novel_author": ("strong", {"class": "author-info__username"}),
137
+ "chapter_list": ("ul", {"class": "table-of-contents"}),
138
+ "chapter_links": ("a", {"href": re.compile(r"/story/\d+.+")}),
139
+ "chapter_title": ("h2", {"class": "part-title"}),
140
+ "content_sel": ("pre", {"class": ""}),
141
+ "next_page": ("a", {"class": "next-part-link"}),
142
+ "delay": 1.5,
143
+ },
144
+ # ── Moonquill ────────────────────────────
145
+ "moonquill.com": {
146
+ "name": "Moonquill",
147
+ "novel_title": ("h1", {"class": "title"}),
148
+ "novel_author": ("a", {"class": "author"}),
149
+ "chapter_list": ("div", {"class": "table-of-contents"}),
150
+ "chapter_links": ("a", {"href": re.compile(r"/book/.+")}),
151
+ "chapter_title": ("h1", {"class": "title"}),
152
+ "content_sel": ("div", {"class": "chapter-body"}),
153
+ "next_page": None,
154
+ "delay": 1.0,
155
+ },
156
+ }
157
 
158
+ # ─────────────────────────────────────────────
159
+ # GENERIC FALLBACK HEURISTICS
160
+ # ─────────────────────────────────────────────
161
+ GENERIC_CONTENT_SELECTORS = [
162
+ ("div", {"id": "chapter-content"}),
163
+ ("div", {"class": "chapter-content"}),
164
+ ("div", {"class": "content"}),
165
+ ("div", {"class": "entry-content"}),
166
+ ("div", {"id": "content"}),
167
+ ("article", {}),
168
+ ("div", {"class": "post-content"}),
169
+ ("div", {"class": "text-left"}),
170
+ ("div", {"class": "novel-content"}),
171
+ ("div", {"class": "read-content"}),
172
+ ]
173
+
174
+ GENERIC_TITLE_SELECTORS = [
175
+ ("h1", {}), ("h2", {}), ("h3", {"class": "chapter-title"}),
176
+ ("title", {}),
177
+ ]
178
+
179
+ # ─────────────────────────────────────────────
180
+ # HELPER UTILITIES
181
+ # ─────────────────────────────────────────────
182
+
183
+ def get_domain(url: str) -> str:
184
+ parsed = urlparse(url)
185
+ domain = parsed.netloc.lower().replace("www.", "")
186
+ return domain
187
+
188
+
189
+ def safe_get(url: str, retries: int = 3, delay: float = 1.0):
190
+ for attempt in range(retries):
191
+ try:
192
+ resp = SESSION.get(url, timeout=20)
193
+ if resp.status_code == 200:
194
+ return resp
195
+ if resp.status_code == 429:
196
+ time.sleep(delay * (attempt + 1) * 3)
197
+ else:
198
+ time.sleep(delay)
199
+ except Exception as e:
200
+ time.sleep(delay * (attempt + 1))
201
+ return None
202
+
203
+
204
+ def extract_text_blocks(soup_element) -> str:
205
+ """Extract clean paragraphs from a BS4 element."""
206
+ if soup_element is None:
207
+ return ""
208
+ # Remove script/style/ads
209
+ for tag in soup_element(["script", "style", "noscript",
210
+ "iframe", "ins", "aside"]):
211
+ tag.decompose()
212
+ # Build paragraphs
213
+ paras = []
214
+ for p in soup_element.find_all(["p", "br"]):
215
+ text = p.get_text(separator=" ", strip=True)
216
+ if text:
217
+ paras.append(text)
218
+ if not paras:
219
+ # fallback: just grab all text
220
+ raw = soup_element.get_text(separator="\n", strip=True)
221
+ paras = [line for line in raw.splitlines() if line.strip()]
222
+ return "\n\n".join(paras)
223
+
224
+
225
+ def clean_title(title: str) -> str:
226
+ return re.sub(r'[<>:"/\\|?*\x00-\x1F]', '', title).strip()
227
+
228
+
229
+ # ─────────────────────────────────────────────
230
+ # CORE SCRAPER
231
+ # ─────────────────────────────────────────────
232
+
233
+ def detect_profile(domain: str):
234
+ for key, profile in SITE_PROFILES.items():
235
+ if key in domain:
236
+ return profile
237
+ return None
238
+
239
+
240
+ def get_toc_links(main_url: str, soup: BeautifulSoup,
241
+ profile, log_fn) -> list:
242
+ """Return list of (chapter_title, chapter_url)."""
243
+ domain = get_domain(main_url)
244
+ base = f"{urlparse(main_url).scheme}://{urlparse(main_url).netloc}"
245
  links = []
246
 
247
+ # ── ScribbleHub uses AJAX for TOC ────────
248
+ if profile and "api_toc" in profile and "scribblehub" in domain:
249
+ fic_id_match = re.search(r"scribblehub\.com/series/(\d+)/", main_url)
250
+ if fic_id_match:
251
+ fic_id = fic_id_match.group(1)
252
+ data = {"action": "wi_gettocchp",
253
+ "strSID": fic_id,
254
+ "strmypostid": "0",
255
+ "strFic": "yes"}
256
+ resp = SESSION.post(profile["api_toc"], data=data, timeout=20)
257
+ if resp.ok:
258
+ toc_soup = BeautifulSoup(resp.text, "html.parser")
259
+ for a in reversed(toc_soup.find_all("a")):
260
+ href = a.get("href", "")
261
+ title = a.get_text(strip=True)
262
+ if href and title:
263
+ links.append((title, href))
264
+ log_fn(f" βœ… ScribbleHub AJAX TOC: {len(links)} chapters")
265
+ return links
266
+
267
+ # ── Standard: find chapter list container ─
268
+ container = None
269
+ if profile and profile.get("chapter_list"):
270
+ tag, attrs = profile["chapter_list"]
271
+ container = soup.find(tag, attrs) if attrs else soup.find(tag)
272
+
273
+ search_root = container if container else soup
274
+ link_sel = (profile.get("chapter_links") if profile else None) or \
275
+ ("a", {"href": re.compile(r"chapter|chap|ch-\d+", re.I)})
276
+
277
+ tag, attrs = link_sel
278
+ found = search_root.find_all(tag, attrs)
279
 
280
+ seen = set()
281
+ for a in found:
282
+ href = a.get("href", "")
283
+ if not href or href in seen:
284
+ continue
285
+ seen.add(href)
286
+ full_url = href if href.startswith("http") else urljoin(base, href)
287
+ title = a.get_text(strip=True) or f"Chapter {len(links)+1}"
288
+ links.append((title, full_url))
289
 
290
+ log_fn(f" πŸ“‹ Found {len(links)} chapter links")
291
  return links
292
 
293
 
294
+ def scrape_chapter(url: str, profile, log_fn) -> tuple:
295
+ """Returns (chapter_title, chapter_text)."""
296
+ resp = safe_get(url, delay=profile.get("delay", 1.2) if profile else 1.2)
297
+ if not resp:
298
+ log_fn(f" ⚠️ Failed to fetch: {url}")
299
+ return None, None
300
+
301
+ soup = BeautifulSoup(resp.text, "html.parser")
302
+
303
+ # ── Title ──
304
+ chapter_title = ""
305
+ if profile and profile.get("chapter_title"):
306
+ t, a = profile["chapter_title"]
307
+ el = soup.find(t, a)
308
+ if el:
309
+ chapter_title = el.get_text(strip=True)
310
+ if not chapter_title:
311
+ for t, a in GENERIC_TITLE_SELECTORS:
312
+ el = soup.find(t, a)
313
+ if el:
314
+ chapter_title = el.get_text(strip=True)
315
+ break
316
+
317
+ # ── Content ──
318
+ content_el = None
319
+ if profile and profile.get("content_sel"):
320
+ t, a = profile["content_sel"]
321
+ content_el = soup.find(t, a)
322
+
323
+ if not content_el:
324
+ for t, a in GENERIC_CONTENT_SELECTORS:
325
+ content_el = soup.find(t, a)
326
+ if content_el:
327
+ break
328
+
329
+ if not content_el:
330
+ # Last resort: largest <div> by text length
331
+ divs = soup.find_all("div")
332
+ if divs:
333
+ content_el = max(divs, key=lambda d: len(d.get_text()))
334
+
335
+ text = extract_text_blocks(content_el)
336
+ return chapter_title, text
337
+
338
+
339
+ # ─────────────────────────────────────────────
340
+ # MAIN ORCHESTRATOR
341
+ # ─────────────────────────────────────────────
342
+
343
+ def scrape_novel(url: str,
344
+ max_chapters: int,
345
+ include_metadata: bool,
346
+ chapter_delay: float,
347
+ progress=gr.Progress()):
348
  logs = []
349
+ chapters_data = []
350
 
351
  def log(msg):
352
  logs.append(msg)
353
+ return "\n".join(logs)
354
+
355
+ url = url.strip()
356
+ if not url.startswith("http"):
357
+ url = "https://" + url
358
+
359
+ domain = get_domain(url)
360
+ profile = detect_profile(domain)
361
+ site_name = profile["name"] if profile else domain
362
+
363
+ yield (None, f"πŸ” Detected site: **{site_name}**\n"
364
+ f"🌐 URL: {url}\n"
365
+ f"{'βœ… Known site profile loaded' if profile else '⚠️ Unknown site β€” using generic heuristics'}\n")
366
+
367
+ # ── Fetch main/TOC page ──────────────────
368
+ log(f"\nπŸ“₯ Fetching novel index page...")
369
+ resp = safe_get(url)
370
+ if not resp:
371
+ yield (None, log("❌ Could not reach the URL. Check your link."))
372
+ return
373
+
374
+ soup = BeautifulSoup(resp.text, "html.parser")
375
+
376
+ # ── Novel title & author ─────────────────
377
+ novel_title = "Unknown Novel"
378
+ novel_author = "Unknown Author"
379
+
380
+ if profile and profile.get("novel_title"):
381
+ t, a = profile["novel_title"]
382
+ el = soup.find(t, a)
383
+ if el:
384
+ novel_title = el.get_text(strip=True)
385
+ if not novel_title or novel_title == "Unknown Novel":
386
+ h1 = soup.find("h1")
387
+ if h1:
388
+ novel_title = h1.get_text(strip=True)
389
+
390
+ if profile and profile.get("novel_author"):
391
+ t, a = profile["novel_author"]
392
+ el = soup.find(t, a)
393
+ if el:
394
+ novel_author = el.get_text(strip=True)
395
+
396
+ novel_title = clean_title(novel_title)
397
+ log(f"πŸ“– Title: {novel_title}")
398
+ log(f"✍️ Author: {novel_author}")
399
+
400
+ yield (None, "\n".join(logs))
401
+
402
+ # ── Gather chapter links ─────────────────
403
+ log("\nπŸ”Ž Collecting chapter list...")
404
+ yield (None, "\n".join(logs))
405
+
406
+ toc_links = get_toc_links(url, soup, profile, log)
407
+ yield (None, "\n".join(logs))
408
+
409
+ if not toc_links:
410
+ # Maybe the URL IS already a chapter β€” scrape just that
411
+ log("⚠️ No TOC found. Attempting to scrape URL as single chapter...")
412
+ ch_title, ch_text = scrape_chapter(url, profile, log)
413
+ if ch_text:
414
+ toc_links = [(ch_title or "Chapter 1", url)]
415
+ else:
416
+ yield (None, log("❌ No chapters found and single-page scrape failed."))
417
+ return
418
+
419
+ # ── Respect max_chapters cap ─────────────
420
+ if max_chapters > 0:
421
+ toc_links = toc_links[:max_chapters]
422
+ log(f"πŸ”’ Capped to {max_chapters} chapters (as requested).")
423
+
424
+ total = len(toc_links)
425
+ log(f"\nπŸš€ Starting scrape of {total} chapters...\n{'─'*40}")
426
+ yield (None, "\n".join(logs))
427
+
428
+ delay = chapter_delay if chapter_delay > 0 else (
429
+ profile.get("delay", 1.2) if profile else 1.2
 
 
 
 
 
 
 
 
 
430
  )
431
 
432
+ # ── Scrape each chapter ──────────────────
433
+ for idx, (ch_link_title, ch_url) in enumerate(toc_links, 1):
434
+ progress((idx - 1) / total,
435
+ desc=f"Chapter {idx}/{total}")
436
+
437
+ log(f"[{idx:>4}/{total}] {ch_link_title[:60]}")
438
+ ch_title, ch_text = scrape_chapter(ch_url, profile, log)
439
+
440
+ display_title = ch_title or ch_link_title or f"Chapter {idx}"
441
+ if not ch_text:
442
+ log(f" ⚠️ Empty content β€” skipping")
443
+ chapters_data.append((display_title, "(Content unavailable)"))
444
+ else:
445
+ word_count = len(ch_text.split())
446
+ log(f" βœ… {word_count:,} words")
447
+ chapters_data.append((display_title, ch_text))
448
+
449
+ yield (None, "\n".join(logs))
450
+ time.sleep(delay)
451
+
452
+ # ── Build output TXT ─────────────────────
453
+ log(f"\n{'─'*40}")
454
+ log("πŸ“ Building output file...")
455
+
456
+ lines = []
457
+
458
+ if include_metadata:
459
+ lines += [
460
+ "═" * 60,
461
+ f" {novel_title}",
462
+ f" by {novel_author}",
463
+ "═" * 60,
464
+ f" Source: {url}",
465
+ f" Site: {site_name}",
466
+ f" Chapters: {len(chapters_data)}",
467
+ f" Total words: {sum(len(c[1].split()) for c in chapters_data):,}",
468
+ "═" * 60,
469
+ "",
470
+ ]
471
+
472
+ for i, (ch_title, ch_text) in enumerate(chapters_data, 1):
473
+ lines += [
474
+ "",
475
+ "─" * 60,
476
+ f" Chapter {i}: {ch_title}",
477
+ "─" * 60,
478
+ "",
479
+ ch_text,
480
+ "",
481
+ ]
482
+
483
+ full_text = "\n".join(lines)
484
+
485
+ # ── Save to temp file ────────────────────
486
+ safe_name = re.sub(r'\s+', '_', clean_title(novel_title))[:80]
487
+ tmp = tempfile.NamedTemporaryFile(
488
+ mode="w", encoding="utf-8",
489
+ suffix=".txt",
490
+ prefix=f"{safe_name}_",
491
+ delete=False
492
  )
493
+ tmp.write(full_text)
494
+ tmp.close()
495
+
496
+ total_words = sum(len(c[1].split()) for c in chapters_data)
497
+ log(f"\nπŸŽ‰ DONE!")
498
+ log(f" πŸ“š {len(chapters_data)} chapters scraped")
499
+ log(f" πŸ“ {total_words:,} total words")
500
+ log(f" πŸ’Ύ File ready for download!")
501
+
502
+ yield (tmp.name, "\n".join(logs))
503
+
504
+
505
+ # ─────────────────────────────────────────────
506
+ # GRADIO UI
507
+ # ─────────────────────────────────────────────
508
+
509
+ SUPPORTED_SITES = "\n".join(
510
+ f"β€’ {p['name']} ({k})" for k, p in SITE_PROFILES.items()
511
+ )
512
+
513
+ THEME = gr.themes.Base(
514
+ primary_hue="violet",
515
+ secondary_hue="purple",
516
+ neutral_hue="slate",
517
+ font=[gr.themes.GoogleFont("Syne"), gr.themes.GoogleFont("JetBrains Mono")],
518
+ ).set(
519
+ body_background_fill="#0f0f13",
520
+ body_text_color="#e8e6f0",
521
+ block_background_fill="#1a1a24",
522
+ block_border_color="#2d2d40",
523
+ input_background_fill="#12121a",
524
+ button_primary_background_fill="*primary_500",
525
+ )
526
+
527
+ CSS = """
528
+ body { background: #0f0f13 !important; }
529
+ .title-block {
530
+ text-align: center;
531
+ padding: 2rem 0 1rem;
532
+ background: linear-gradient(135deg, #1a0530 0%, #0d1a3a 50%, #1a0530 100%);
533
+ border-radius: 16px;
534
+ margin-bottom: 1rem;
535
+ border: 1px solid #2a1a4a;
536
+ }
537
+ .title-block h1 {
538
+ font-size: 2.4rem;
539
+ background: linear-gradient(90deg, #b06ef3, #6ea8f7, #b06ef3);
540
+ background-size: 200%;
541
+ -webkit-background-clip: text;
542
+ -webkit-text-fill-color: transparent;
543
+ animation: shimmer 3s linear infinite;
544
+ font-family: 'Syne', sans-serif;
545
+ font-weight: 800;
546
+ letter-spacing: -1px;
547
+ margin-bottom: 0.3rem;
548
+ }
549
+ @keyframes shimmer { 0%{background-position:0%} 100%{background-position:200%} }
550
+ .title-block p { color: #9090b0; font-size: 0.95rem; margin: 0; }
551
+ .log-box textarea {
552
+ font-family: 'JetBrains Mono', monospace !important;
553
+ font-size: 0.78rem !important;
554
+ line-height: 1.6 !important;
555
+ color: #a0e8a0 !important;
556
+ background: #050a05 !important;
557
+ }
558
+ .site-list {
559
+ background: #12121a;
560
+ border: 1px solid #2a2a3a;
561
+ border-radius: 8px;
562
+ padding: 1rem 1.2rem;
563
+ font-size: 0.82rem;
564
+ color: #8080a0;
565
+ line-height: 1.9;
566
+ }
567
+ .gr-button-primary {
568
+ background: linear-gradient(135deg, #7c3aed, #3b82f6) !important;
569
+ border: none !important;
570
+ font-size: 1rem !important;
571
+ font-weight: 700 !important;
572
+ letter-spacing: 0.5px;
573
+ transition: all 0.2s !important;
574
+ }
575
+ .gr-button-primary:hover {
576
+ transform: translateY(-1px) !important;
577
+ box-shadow: 0 4px 20px rgba(124, 58, 237, 0.5) !important;
578
+ }
579
+ """
580
 
581
+ with gr.Blocks(theme=THEME, css=CSS, title="πŸ“š WebNovel Scraper") as demo:
582
+
583
+ gr.HTML("""
584
+ <div class="title-block">
585
+ <h1>πŸ“š WebNovel Scraper</h1>
586
+ <p>Paste any novel URL Β· Get a clean .txt file Β· Works on 10+ sites</p>
587
+ </div>
588
+ """)
589
+
590
+ with gr.Row():
591
+ with gr.Column(scale=3):
592
+ url_input = gr.Textbox(
593
+ label="πŸ”— Novel URL",
594
+ placeholder="https://www.royalroad.com/fiction/12345/your-novel-title",
595
+ lines=1,
596
+ )
597
+
598
+ with gr.Row():
599
+ max_ch = gr.Slider(
600
+ label="πŸ“¦ Max Chapters (0 = all)",
601
+ minimum=0, maximum=1000, step=1, value=0,
602
+ )
603
+ delay_sl = gr.Slider(
604
+ label="⏱️ Delay Between Requests (sec)",
605
+ minimum=0.5, maximum=5.0, step=0.1, value=1.5,
606
+ )
607
+
608
+ meta_cb = gr.Checkbox(
609
+ label="πŸ“‹ Include Metadata Header in Output",
610
+ value=True,
611
+ )
612
+
613
+ scrape_btn = gr.Button("⚑ SCRAPE NOVEL", variant="primary", size="lg")
614
+
615
+ file_out = gr.File(
616
+ label="πŸ“₯ Download Scraped Novel (.txt)",
617
+ visible=True,
618
+ interactive=False,
619
+ )
620
+
621
+ with gr.Column(scale=2):
622
+ gr.HTML(f"""
623
+ <div class="site-list">
624
+ <strong style="color:#b06ef3;">βœ… Supported Sites</strong><br><br>
625
+ {"<br>".join(f"β€’ <span style='color:#c0c0e0'>{p['name']}</span> <span style='opacity:0.5;font-size:0.75rem'>({k})</span>" for k, p in SITE_PROFILES.items())}
626
+ <br><br>
627
+ <span style="color:#6060a0;font-size:0.8rem;">
628
+ βš™οΈ <strong>Generic mode</strong> activates automatically for unlisted sites β€” heuristics detect chapters & content on most novel platforms.
629
+ </span>
630
+ </div>
631
+ """)
632
+
633
+ log_out = gr.Textbox(
634
+ label="πŸ“œ Scraping Log",
635
+ lines=18,
636
+ max_lines=30,
637
+ interactive=False,
638
+ elem_classes=["log-box"],
639
+ placeholder="Logs will appear here once you start scraping...",
640
  )
641
 
642
+ gr.HTML("""
643
+ <div style="text-align:center;color:#4a4a6a;font-size:0.78rem;margin-top:0.5rem;padding-top:0.5rem;border-top:1px solid #1e1e2e;">
644
+ ⚠️ Please respect each site's Terms of Service. Use responsibly and avoid hammering servers.
645
+ Built with ❀️ using Python + Gradio + BeautifulSoup4.
646
+ </div>
647
+ """)
648
+
649
+ scrape_btn.click(
650
+ fn=scrape_novel,
651
+ inputs=[url_input, max_ch, meta_cb, delay_sl],
652
+ outputs=[file_out, log_out],
653
  )
654
 
655
+ if __name__ == "__main__":
656
+ demo.launch()