Krixter commited on
Commit
351e9ae
Β·
verified Β·
1 Parent(s): dfbb94a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +316 -132
app.py CHANGED
@@ -1,144 +1,328 @@
1
  import gradio as gr
2
- from huggingface_hub import InferenceClient
3
  import requests
4
  from bs4 import BeautifulSoup
 
5
  import time
6
- import os
 
7
 
8
- # Initialize the Hugging Face client pointing to Qwen 2.5 72B
9
- # It uses the token automatically provided by the Hugging Face Space environment
10
- client = InferenceClient("Qwen/Qwen2.5-72B-Instruct")
11
 
12
- def scrape_url(url):
13
- """Simple web scraper to extract text from a novel page."""
14
- try:
15
- headers = {
16
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
17
- }
18
- response = requests.get(url, headers=headers, timeout=15)
19
- response.raise_for_status()
20
-
21
- soup = BeautifulSoup(response.text, 'html.parser')
22
-
23
- # Try to guess common text containers for web novels
24
- # Drops scripts, styles, and navigational footers
25
- for element in soup(["script", "style", "nav", "footer", "header"]):
26
- element.extract()
27
-
28
- # Common body containers for light/web novel sites
29
- text_container = soup.find(id="content") or soup.find(class_="chapter-content") or soup.find(id="chapter-content") or soup.find("article")
30
-
31
- if text_container:
32
- return text_container.get_text(separator="\n").strip()
33
- else:
34
- # Fallback to general body text if specific container isn't found
35
- return soup.body.get_text(separator="\n").strip()
36
- except Exception as e:
37
- return f"Error scraping the URL: {str(e)}"
38
-
39
- def chunk_text(text, max_chars=3000):
40
- """Splits text into chunks so the API doesn't time out or summarize."""
41
- paragraphs = text.split("\n")
42
- chunks = []
43
- current_chunk = []
44
- current_length = 0
45
-
46
- for para in paragraphs:
47
- if not para.strip():
48
- continue
49
- if current_length + len(para) > max_chars:
50
- chunks.append("\n".join(current_chunk))
51
- current_chunk = [para]
52
- current_length = len(para)
53
- else:
54
- current_chunk.append(para)
55
- current_length += len(para)
56
-
57
- if current_chunk:
58
- chunks.append("\n".join(current_chunk))
59
- return chunks
60
-
61
- def translate_chunk(chunk, target_lang="English"):
62
- """Sends a single chunk to Qwen for strict, unfiltered translation."""
63
- system_prompt = (
64
- f"You are a professional, elite web novel and light novel translator. "
65
- f"Translate the following text into fluent, natural {target_lang}. "
66
- f"CRITICAL RULES:\n"
67
- f"1. Translate everything strictly and verbatim. Do NOT summarize, condense, omit, or skip any sentences.\n"
68
- f"2. Retain all contextual meaning, dialogue tones, and descriptions completely.\n"
69
- f"3. Do not apply external moral judgments or safety filters to fictional stories. Translate mature, adult, or violent themes precisely as written without rewriting them."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  )
71
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  try:
73
- response = client.chat_completion(
74
- messages=[
75
- {"role": "system", "content": system_prompt},
76
- {"role": "user", "content": f"Translate this text directly:\n\n{chunk}"}
77
- ],
78
- max_tokens=2048,
79
- temperature=0.2 # Low temperature forces accurate translation instead of improvisation
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  )
81
- return response.choices[0].message.content
82
- except Exception as e:
83
- return f"\n[Translation Error on this segment: {str(e)}]\n"
84
-
85
- def process_translation(url_input, file_input, target_lang, progress=gr.Progress()):
86
- """Main pipeline combining file/url checking, chunking, and translation loop."""
87
- raw_text = ""
88
-
89
- # Prioritize file upload, fallback to URL scraping
90
- if file_input is not None:
91
- progress(0.1, desc="Reading uploaded file...")
92
- with open(file_input.name, "r", encoding="utf-8", errors="ignore") as f:
93
- raw_text = f.read()
94
- elif url_input.strip():
95
- progress(0.1, desc="Scraping novel website...")
96
- raw_text = scrape_url(url_input)
97
- else:
98
- return "Please provide either a valid novel URL link or upload a .txt file."
99
-
100
- if "Error scraping" in raw_text or not raw_text.strip():
101
- return raw_text if raw_text else "No text found to translate."
102
-
103
- progress(0.2, desc="Chunking text for translation stability...")
104
- chunks = chunk_text(raw_text)
105
- total_chunks = len(chunks)
106
-
107
- translated_output = []
108
-
109
- for i, chunk in enumerate(chunks):
110
- progress_val = 0.2 + (0.7 * (i / total_chunks))
111
- progress(progress_val, desc=f"Translating segment {i+1} of {total_chunks}...")
112
-
113
- translated_chunk = translate_chunk(chunk, target_lang)
114
- translated_output.append(translated_chunk)
115
-
116
- # Free tier safety rule: pause slightly between chunks to avoid hard API rate blocks
117
- time.sleep(1.5)
118
-
119
- progress(1.0, desc="Translation complete!")
120
- return "\n\n".join(translated_output)
121
-
122
- # Building the interactive Gradio Interface
123
- with gr.Blocks(theme=gr.themes.Soft()) as demo:
124
- gr.Markdown("# πŸ“– Uncensored Web Novel & Text Translator")
125
- gr.Markdown("Translate raw web chapters or local text files directly using high-fidelity AI processing.")
126
-
127
  with gr.Row():
128
- with gr.Column():
129
- url_box = gr.Textbox(label="Option A: Paste Raw Novel URL", placeholder="https://example-raw-novel-site.com/chapter-1")
130
- file_box = gr.File(label="Option B: Upload a .txt file instead", file_types=[".txt"])
131
- lang_dropdown = gr.Dropdown(choices=["English", "Spanish", "French", "German", "Russian"], value="English", label="Target Language")
132
- submit_btn = gr.Button("Start Translation", variant="primary")
133
-
134
- with gr.Column():
135
- output_box = gr.Textbox(label="Translated Output", lines=25, show_copy_button=True)
136
-
137
- submit_btn.click(
138
- fn=process_translation,
139
- inputs=[url_box, file_box, lang_dropdown],
140
- outputs=output_box
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
  )
142
 
143
- if __name__ == "__main__":
144
- demo.launch()
 
1
  import gradio as gr
 
2
  import requests
3
  from bs4 import BeautifulSoup
4
+ import re
5
  import time
6
+ from urllib.parse import urlparse
7
+ from pathlib import Path
8
 
9
+ # ──────────────────────────────────────────────
10
+ # HTTP helpers
11
+ # ──────────────────────────────────────────────
12
 
13
+ HEADERS = {
14
+ "User-Agent": (
15
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
16
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
17
+ "Chrome/124.0.0.0 Safari/537.36"
18
+ ),
19
+ "Accept-Language": "en-US,en;q=0.9",
20
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
21
+ "Referer": "https://novelfire.net/",
22
+ }
23
+
24
+ SESSION = requests.Session()
25
+ SESSION.headers.update(HEADERS)
26
+
27
+
28
+ def fetch_html(url: str, retries: int = 3, timeout: int = 20) -> str:
29
+ for attempt in range(1, retries + 1):
30
+ try:
31
+ resp = SESSION.get(url, timeout=timeout)
32
+ resp.raise_for_status()
33
+ return resp.text
34
+ except requests.RequestException as exc:
35
+ if attempt == retries:
36
+ raise RuntimeError(f"Failed to fetch {url} after {retries} attempts: {exc}")
37
+ time.sleep(2 ** attempt)
38
+
39
+
40
+ # ──────────────────────────────────────────────
41
+ # Chapter-list helpers
42
+ # ──────────────────────────────────────────────
43
+
44
+ def base_chapters_url(raw_url: str) -> str:
45
+ raw_url = raw_url.strip().rstrip("/").split("?")[0]
46
+ raw_url = re.sub(r"/chapter-\d+$", "", raw_url)
47
+ if not raw_url.endswith("/chapters"):
48
+ raw_url = raw_url.rstrip("/") + "/chapters"
49
+ return raw_url
50
+
51
+
52
+ def extract_novel_title(soup: BeautifulSoup) -> str:
53
+ tag = soup.select_one("h1 a, header h1, .novel-item h1 a")
54
+ if tag:
55
+ return tag.get_text(strip=True)
56
+ title_tag = soup.select_one("title")
57
+ if title_tag:
58
+ raw = title_tag.get_text(strip=True)
59
+ return re.split(r"\s*(?:Novel Chapters|–|-)\s*", raw)[0].strip()
60
+ return "Novel"
61
+
62
+
63
+ def get_all_chapter_links(chapters_base_url: str, log_fn=None) -> list:
64
+ all_chapters = []
65
+ page = 1
66
+
67
+ while True:
68
+ page_url = f"{chapters_base_url}?page={page}" if page > 1 else chapters_base_url
69
+ if log_fn:
70
+ log_fn(f"πŸ“„ Fetching chapter list page {page}: {page_url}")
71
+ html = fetch_html(page_url)
72
+ soup = BeautifulSoup(html, "html.parser")
73
+
74
+ items = soup.select("ul.chapter-list li a")
75
+ if not items:
76
+ break
77
+
78
+ for a in items:
79
+ href = a.get("href", "")
80
+ if not href or "/chapter-" not in href:
81
+ continue
82
+ if href.startswith("/"):
83
+ parsed = urlparse(chapters_base_url)
84
+ href = f"{parsed.scheme}://{parsed.netloc}{href}"
85
+ m = re.search(r"/chapter-(\d+)", href)
86
+ num = int(m.group(1)) if m else 0
87
+ title_text = a.get("title") or a.get_text(strip=True)
88
+ title_text = re.sub(r"^\d+\s*", "", title_text)
89
+ title_text = re.sub(
90
+ r"\d+\s*(second|minute|hour|day|week|month|year)s?\s*ago.*$",
91
+ "", title_text, flags=re.I
92
+ ).strip()
93
+ all_chapters.append({"url": href, "title": title_text, "num": num})
94
+
95
+ # Check for a next page link
96
+ next_page_exists = False
97
+ for pg_a in soup.select(".pagination a.page-link"):
98
+ href_pg = pg_a.get("href", "")
99
+ m2 = re.search(r"page=(\d+)", href_pg)
100
+ if m2 and int(m2.group(1)) > page:
101
+ next_page_exists = True
102
+ break
103
+ if not next_page_exists:
104
+ break
105
+ page += 1
106
+
107
+ # Deduplicate & sort
108
+ seen, unique = set(), []
109
+ for ch in all_chapters:
110
+ if ch["num"] not in seen:
111
+ seen.add(ch["num"])
112
+ unique.append(ch)
113
+ unique.sort(key=lambda x: x["num"])
114
+ return unique
115
+
116
+
117
+ # ──────────────────────────────────────────────
118
+ # Chapter content extraction
119
+ # ──────────────────────────────────────────────
120
+
121
+ def extract_chapter_content(html: str):
122
+ soup = BeautifulSoup(html, "html.parser")
123
+
124
+ title_tag = soup.select_one("span.chapter-title, h1.chapter-title, .chapter-title")
125
+ title = title_tag.get_text(strip=True) if title_tag else ""
126
+
127
+ content_div = soup.select_one("#content") or soup.select_one(
128
+ "#chapter-container, .d-chapter-content"
129
  )
130
+ if not content_div:
131
+ return title, ""
132
+
133
+ for junk in content_div.select(
134
+ "[id*='bg-ssp'], .nf-ads, iframe, script, style, "
135
+ "#restore-scroll-btn, .text-center, noscript, [class*='ads'], "
136
+ "[id*='ads'], [id*='ad-']"
137
+ ):
138
+ junk.decompose()
139
+
140
+ paragraphs = [
141
+ tag.get_text(separator=" ", strip=True)
142
+ for tag in content_div.find_all("p")
143
+ if tag.get_text(strip=True)
144
+ ]
145
+ if not paragraphs:
146
+ raw = content_div.get_text(separator="\n", strip=True)
147
+ paragraphs = [ln.strip() for ln in raw.splitlines() if ln.strip()]
148
+
149
+ return title, "\n".join(paragraphs)
150
+
151
+
152
+ def safe_filename(s: str) -> str:
153
+ s = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", s)
154
+ return s.strip("._ ")[:120]
155
+
156
+
157
+ # ──────────────────────────────────────────────
158
+ # Scraper – generator so Gradio streams updates
159
+ # ──────────────────────────────────────────────
160
+
161
+ # HF Spaces: use /tmp for all I/O (writable, ephemeral)
162
+ TMP_ROOT = Path("/tmp/novel_scraper")
163
+ TMP_ROOT.mkdir(parents=True, exist_ok=True)
164
+
165
+
166
+ def run_scraper(url: str, ch_start: int, ch_end: int, rps: float, progress=gr.Progress()):
167
+ logs: list[str] = []
168
+
169
+ def log(msg: str) -> str:
170
+ logs.append(msg)
171
+ return "\n".join(logs[-80:])
172
+
173
+ def emit(extra_log: str | None = None):
174
+ if extra_log:
175
+ log(extra_log)
176
+ return "\n".join(logs[-80:]), gr.update(visible=False)
177
+
178
  try:
179
+ url = url.strip()
180
+ if not url:
181
+ yield log("❌ Please enter a URL."), gr.update(visible=False)
182
+ return
183
+
184
+ chapters_base = base_chapters_url(url)
185
+ yield emit(f"πŸ“– Chapters base URL: {chapters_base}")
186
+
187
+ # ── 1. Collect all chapter links ──────────────────────
188
+ progress(0.0, desc="Discovering chapters…")
189
+ all_chapters = get_all_chapter_links(chapters_base, log_fn=log)
190
+ yield emit()
191
+
192
+ if not all_chapters:
193
+ yield emit("❌ No chapters found. Double-check the URL.")
194
+ return
195
+
196
+ html_first = fetch_html(chapters_base)
197
+ novel_title = extract_novel_title(BeautifulSoup(html_first, "html.parser"))
198
+ yield emit(f"πŸ“š Novel: {novel_title} | {len(all_chapters)} chapters total")
199
+
200
+ # ── 2. Apply chapter range ────────────────────────────
201
+ nums = [c["num"] for c in all_chapters]
202
+ actual_start = ch_start if ch_start > 0 else min(nums)
203
+ actual_end = ch_end if ch_end > 0 else max(nums)
204
+
205
+ selected = [c for c in all_chapters if actual_start <= c["num"] <= actual_end]
206
+ if not selected:
207
+ yield emit(f"❌ No chapters in range {actual_start}–{actual_end}.")
208
+ return
209
+
210
+ yield emit(
211
+ f"🎯 Chapters {actual_start}–{actual_end} "
212
+ f"({len(selected)} chapters) @ {rps} req/s"
213
+ )
214
+
215
+ # ── 3. Prepare cache folder ───────────────────────────
216
+ safe_novel = safe_filename(novel_title)
217
+ html_folder = TMP_ROOT / safe_novel / "html_cache"
218
+ html_folder.mkdir(parents=True, exist_ok=True)
219
+ delay = 1.0 / max(rps, 0.1)
220
+
221
+ # ── 4. Download chapters ──────────────────────────────
222
+ compiled: list[tuple[int, str, str]] = []
223
+ total = len(selected)
224
+
225
+ for idx, ch in enumerate(selected, start=1):
226
+ progress(idx / total * 0.85, desc=f"Chapter {ch['num']} / {actual_end}…")
227
+ html_path = html_folder / f"chapter_{ch['num']:05d}.html"
228
+
229
+ if html_path.exists():
230
+ yield emit(f" ♻️ [{idx}/{total}] Ch {ch['num']} (cached)")
231
+ html_text = html_path.read_text(encoding="utf-8")
232
+ else:
233
+ try:
234
+ html_text = fetch_html(ch["url"])
235
+ html_path.write_text(html_text, encoding="utf-8")
236
+ yield emit(f" βœ… [{idx}/{total}] Ch {ch['num']}: {ch['title']}")
237
+ except RuntimeError as exc:
238
+ yield emit(f" ❌ [{idx}/{total}] Ch {ch['num']} FAILED: {exc}")
239
+ compiled.append((ch["num"], ch["title"] or f"Chapter {ch['num']}",
240
+ f"[Chapter {ch['num']} could not be downloaded.]"))
241
+ time.sleep(delay)
242
+ continue
243
+
244
+ title, body = extract_chapter_content(html_text)
245
+ if not title:
246
+ title = ch["title"] or f"Chapter {ch['num']}"
247
+ compiled.append((ch["num"], title, body))
248
+ time.sleep(delay)
249
+
250
+ # ── 5. Merge into one TXT ─────────────────────────────
251
+ progress(0.95, desc="Merging…")
252
+ compiled.sort(key=lambda x: x[0])
253
+
254
+ out_name = safe_filename(f"{novel_title} Ch{actual_start}-{actual_end}") + ".txt"
255
+ out_path = TMP_ROOT / safe_novel / out_name
256
+
257
+ with open(out_path, "w", encoding="utf-8") as fout:
258
+ for _, title, body in compiled:
259
+ fout.write(f"{title}\n\n{body}\n\n")
260
+
261
+ progress(1.0, desc="Done!")
262
+ size_kb = out_path.stat().st_size // 1024
263
+ yield (
264
+ log(f"\nβœ… Done! {len(compiled)} chapters | {size_kb} KB"),
265
+ gr.update(value=str(out_path), visible=True),
266
  )
267
+
268
+ except Exception as exc:
269
+ import traceback
270
+ yield log(f"πŸ’₯ Error: {exc}\n{traceback.format_exc()}"), gr.update(visible=False)
271
+
272
+
273
+ # ──────────────────────────────────────────────
274
+ # Gradio UI
275
+ # ──────────────────────────────────────────────
276
+
277
+ with gr.Blocks(title="NovelFire Scraper") as demo:
278
+ gr.Markdown("""
279
+ # πŸ“š NovelFire Chapter Scraper
280
+ Paste **any** NovelFire book URL β€” the home page, a chapter URL, or the `/chapters` listing.
281
+ Set a chapter range (leave both at **0** to grab everything), adjust speed, and hit **Start**.
282
+ All chapters are merged into a single `.txt` file with no dividers between them.
283
+ """)
284
+
285
+ url_input = gr.Textbox(
286
+ label="Book URL",
287
+ placeholder="https://novelfire.net/book/green-tea-shes-clearly-a-hidden-gem",
288
+ lines=1,
289
+ )
290
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
291
  with gr.Row():
292
+ ch_start = gr.Number(
293
+ label="Chapter Start (0 = first available)",
294
+ value=0, minimum=0, precision=0,
295
+ )
296
+ ch_end = gr.Number(
297
+ label="Chapter End (0 = last available)",
298
+ value=0, minimum=0, precision=0,
299
+ )
300
+ rps = gr.Slider(
301
+ label="Requests / second",
302
+ minimum=0.2, maximum=5.0, step=0.1, value=1.0,
303
+ info="↓ slower = kinder to server ↑ faster = quicker download",
304
+ )
305
+
306
+ start_btn = gr.Button("πŸš€ Start Scraping", variant="primary", size="lg")
307
+
308
+ log_box = gr.Textbox(
309
+ label="Live Progress",
310
+ lines=22,
311
+ max_lines=22,
312
+ interactive=False,
313
+ autoscroll=True,
314
+ )
315
+
316
+ dl_file = gr.File(
317
+ label="⬇️ Download your TXT",
318
+ visible=False,
319
+ interactive=False,
320
+ )
321
+
322
+ start_btn.click(
323
+ fn=run_scraper,
324
+ inputs=[url_input, ch_start, ch_end, rps],
325
+ outputs=[log_box, dl_file],
326
  )
327
 
328
+ demo.launch(server_name="0.0.0.0", server_port=7860)