thohan88 commited on
Commit
5734530
·
1 Parent(s): 267e34a

feat: add support for path-based link filtering and mutual exclusion for scraper modes

Browse files
Files changed (2) hide show
  1. tools/scraper.py +49 -7
  2. tools/worker.py +24 -15
tools/scraper.py CHANGED
@@ -97,10 +97,19 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
97
  current_filter = input.link_filter()
98
  last_auto = last_auto_filter.get()
99
 
 
 
 
 
 
 
 
 
 
100
  # Update if currently empty OR if it matches what we last auto-set
101
  if not current_filter or current_filter == last_auto:
102
- ui.update_text("link_filter", value=host, session=session)
103
- last_auto_filter.set(host)
104
 
105
  @reactive.effect
106
  def auto_check_sitemap():
@@ -108,15 +117,39 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
108
  if url and url.strip().lower().endswith("sitemap.xml"):
109
  ui.update_checkbox("sitemap", value=True, session=session)
110
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  @reactive.effect
112
  def update_filter():
113
- path = scrapes_dir / "filter.txt"
114
  regex = input.link_filter()
 
 
 
115
  if regex:
116
  with open(path, "w") as f:
117
  f.write(regex)
118
  elif path.exists():
119
  path.unlink()
 
 
 
 
 
 
 
 
 
 
120
 
121
  @reactive.poll(check_log_mtime, interval_secs=0.1)
122
  def log_content():
@@ -210,6 +243,9 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
210
  scraper_script = Path(__file__).parent / "worker.py"
211
 
212
  cmd = [sys.executable, "-u", str(scraper_script), url, str(db_path), "--log-file", str(log_path)]
 
 
 
213
 
214
 
215
  if input.cloudscraper():
@@ -504,8 +540,8 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
504
  ),
505
  ui.div(
506
  ui.download_button("download_db", "Download Database", class_="btn-success"),
507
- ui.download_button("download_md", "Download Markdown", class_="btn-info"),
508
- ui.download_button("download_combined_md", "Download Combined MD", class_="btn-primary"),
509
  class_="d-flex gap-2 justify-content-center"
510
  ),
511
  class_="d-flex flex-column align-items-center"
@@ -518,7 +554,10 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
518
  if path and path.exists():
519
  return str(path)
520
 
521
- @render.download(filename=lambda: (db_file_path.get().stem + "_combined.md") if db_file_path.get() else "combined.md")
 
 
 
522
  def download_combined_md():
523
  path = db_file_path.get()
524
  if not path or not path.exists():
@@ -552,7 +591,10 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
552
  print(f"Error generating combined markdown: {e}")
553
  return None
554
 
555
- @render.download(filename=lambda: (db_file_path.get().stem + "_markdown.zip") if db_file_path.get() else "markdown.zip")
 
 
 
556
  def download_md():
557
  path = db_file_path.get()
558
  if not path or not path.exists():
 
97
  current_filter = input.link_filter()
98
  last_auto = last_auto_filter.get()
99
 
100
+ # Build filter as host + path (strip trailing slash) so that
101
+ # e.g. https://developers.cloudflare.com/containers/ auto-fills
102
+ # "developers.cloudflare.com/containers" not just the hostname.
103
+ try:
104
+ parsed_path = urlparse(url).path.rstrip("/")
105
+ except Exception:
106
+ parsed_path = ""
107
+ auto_filter = host + parsed_path if parsed_path and parsed_path != "/" else host
108
+
109
  # Update if currently empty OR if it matches what we last auto-set
110
  if not current_filter or current_filter == last_auto:
111
+ ui.update_text("link_filter", value=auto_filter, session=session)
112
+ last_auto_filter.set(auto_filter)
113
 
114
  @reactive.effect
115
  def auto_check_sitemap():
 
117
  if url and url.strip().lower().endswith("sitemap.xml"):
118
  ui.update_checkbox("sitemap", value=True, session=session)
119
 
120
+ @reactive.effect
121
+ @reactive.event(input.cloudscraper)
122
+ def _():
123
+ if input.cloudscraper():
124
+ ui.update_checkbox("selenium", value=False, session=session)
125
+
126
+ @reactive.effect
127
+ @reactive.event(input.selenium)
128
+ def _():
129
+ if input.selenium():
130
+ ui.update_checkbox("cloudscraper", value=False, session=session)
131
+
132
  @reactive.effect
133
  def update_filter():
 
134
  regex = input.link_filter()
135
+
136
+ # Update global filter (for new jobs)
137
+ path = scrapes_dir / "filter.txt"
138
  if regex:
139
  with open(path, "w") as f:
140
  f.write(regex)
141
  elif path.exists():
142
  path.unlink()
143
+
144
+ # Also update current job's filter if one is active
145
+ db_path = db_file_path.get()
146
+ if db_path:
147
+ job_filter_path = db_path.parent / "filter.txt"
148
+ if regex:
149
+ with open(job_filter_path, "w") as f:
150
+ f.write(regex)
151
+ elif job_filter_path.exists():
152
+ job_filter_path.unlink()
153
 
154
  @reactive.poll(check_log_mtime, interval_secs=0.1)
155
  def log_content():
 
243
  scraper_script = Path(__file__).parent / "worker.py"
244
 
245
  cmd = [sys.executable, "-u", str(scraper_script), url, str(db_path), "--log-file", str(log_path)]
246
+
247
+ if input.link_filter():
248
+ cmd.extend(["--link-filter", input.link_filter()])
249
 
250
 
251
  if input.cloudscraper():
 
540
  ),
541
  ui.div(
542
  ui.download_button("download_db", "Download Database", class_="btn-success"),
543
+ ui.download_button("download_md", "Download Markdown (ZIP)", class_="btn-info"),
544
+ ui.download_button("download_combined_md", "Download Combined Markdown", class_="btn-primary"),
545
  class_="d-flex gap-2 justify-content-center"
546
  ),
547
  class_="d-flex flex-column align-items-center"
 
554
  if path and path.exists():
555
  return str(path)
556
 
557
+ @render.download(
558
+ filename=lambda: f"{Path(db_file_path.get()).stem}_combined.md" if db_file_path.get() else "combined.md",
559
+ media_type="text/markdown"
560
+ )
561
  def download_combined_md():
562
  path = db_file_path.get()
563
  if not path or not path.exists():
 
591
  print(f"Error generating combined markdown: {e}")
592
  return None
593
 
594
+ @render.download(
595
+ filename=lambda: f"{Path(db_file_path.get()).stem}_markdown.zip" if db_file_path.get() else "markdown.zip",
596
+ media_type="application/zip"
597
+ )
598
  def download_md():
599
  path = db_file_path.get()
600
  if not path or not path.exists():
tools/worker.py CHANGED
@@ -7,9 +7,9 @@ from bs4 import BeautifulSoup
7
  import os
8
  import argparse
9
  import sys
10
- import re
11
  from pathlib import Path
12
  import tempfile
 
13
  from markitdown import MarkItDown
14
  import logging
15
  import cloudscraper
@@ -40,6 +40,11 @@ def get_host(url):
40
  return None
41
 
42
  def is_same_site(host1, host2):
 
 
 
 
 
43
  if not host1 or not host2:
44
  return False
45
 
@@ -49,16 +54,8 @@ def is_same_site(host1, host2):
49
  if h1 == h2:
50
  return True
51
 
52
- # Handle www. and subdomains
53
- h1_root = h1.removeprefix("www.")
54
- h2_root = h2.lower().removeprefix("www.")
55
-
56
- if h1_root == h2_root:
57
- return True
58
-
59
- # Allow subdomains: check if one is a suffix of the other with a dot
60
- # e.g. news.forsvaret.no is part of forsvaret.no
61
- if h1.endswith("." + h2_root) or h2.endswith("." + h1_root):
62
  return True
63
 
64
  return False
@@ -193,7 +190,7 @@ def store_documents(con, doc_links, source_url, doc_table="documents"):
193
  con.execute(f"INSERT INTO {doc_table} (url, source_page, size) VALUES (?, ?, ?)",
194
  (link, source_url, size))
195
 
196
- def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, keep_pdfs=False, use_cloudscraper=False, use_selenium=False, use_sitemap=False):
197
  start_url = normalize_url(start_url)
198
  base_host = get_host(start_url)
199
 
@@ -209,11 +206,14 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
209
  print(f"--- Scraper Started ---", flush=True)
210
  print(f"Target: {start_url}", flush=True)
211
  print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
 
 
212
  print(f"-----------------------", flush=True)
213
 
214
  con = duckdb.connect(db_path, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
215
  init_db(con)
216
 
 
217
  cloudscraper_session = None
218
  if use_cloudscraper:
219
  try:
@@ -277,6 +277,12 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
277
  while reqs_made < max_reqs:
278
  # Dynamic regex filter
279
  filter_regex = None
 
 
 
 
 
 
280
  filter_path = Path("filter.txt")
281
  if filter_path.exists():
282
  try:
@@ -297,8 +303,10 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
297
  current_url, current_depth = next_url_row
298
  con.execute("DELETE FROM queue WHERE url = ?", (current_url,)) # Remove from queue
299
 
300
- # Check against regex (skip if doesn't match)
301
- # Exception: always allow start_url to ensure we start
 
 
302
  if filter_regex and current_url != start_url and not filter_regex.search(current_url):
303
  # print(f"Skipping {current_url} (regex mismatch)", flush=True)
304
  continue
@@ -513,6 +521,7 @@ if __name__ == "__main__":
513
  parser.add_argument("--max-depth", type=int, default=None, help="Maximum depth to scrape (default: unlimited)")
514
  parser.add_argument("--keep-pdfs", action="store_true", help="Keep PDFs in original format")
515
  parser.add_argument("--log-file", help="Path to log file")
 
516
 
517
  args = parser.parse_args()
518
 
@@ -533,4 +542,4 @@ if __name__ == "__main__":
533
  print("Error: Could not derive database name from URL. Please specify db_path.")
534
  sys.exit(1)
535
 
536
- scrape(args.url, db_path, throttle_rate=args.throttle, max_reqs=args.max_reqs, max_depth=args.max_depth, keep_pdfs=args.keep_pdfs, use_cloudscraper=args.cloudscraper, use_selenium=args.selenium, use_sitemap=args.sitemap)
 
7
  import os
8
  import argparse
9
  import sys
 
10
  from pathlib import Path
11
  import tempfile
12
+ import hashlib
13
  from markitdown import MarkItDown
14
  import logging
15
  import cloudscraper
 
40
  return None
41
 
42
  def is_same_site(host1, host2):
43
+ """Return True only if the two hosts are the exact same host,
44
+ or one is the bare domain and the other is the www. variant.
45
+ Subdomains (e.g. news.example.com vs example.com) are intentionally
46
+ NOT treated as same-site, because that causes cross-subdomain leakage
47
+ when a path-based link_filter is in use."""
48
  if not host1 or not host2:
49
  return False
50
 
 
54
  if h1 == h2:
55
  return True
56
 
57
+ # Allow only the www. <-> bare-domain pair, nothing else
58
+ if h1.removeprefix("www.") == h2.removeprefix("www."):
 
 
 
 
 
 
 
 
59
  return True
60
 
61
  return False
 
190
  con.execute(f"INSERT INTO {doc_table} (url, source_page, size) VALUES (?, ?, ?)",
191
  (link, source_url, size))
192
 
193
+ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, keep_pdfs=False, use_cloudscraper=False, use_selenium=False, use_sitemap=False, link_filter=None):
194
  start_url = normalize_url(start_url)
195
  base_host = get_host(start_url)
196
 
 
206
  print(f"--- Scraper Started ---", flush=True)
207
  print(f"Target: {start_url}", flush=True)
208
  print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
209
+ if link_filter:
210
+ print(f"Link Filter: {link_filter}", flush=True)
211
  print(f"-----------------------", flush=True)
212
 
213
  con = duckdb.connect(db_path, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
214
  init_db(con)
215
 
216
+
217
  cloudscraper_session = None
218
  if use_cloudscraper:
219
  try:
 
277
  while reqs_made < max_reqs:
278
  # Dynamic regex filter
279
  filter_regex = None
280
+ if link_filter:
281
+ try:
282
+ filter_regex = re.compile(link_filter)
283
+ except Exception as e:
284
+ print(f"Error compiling initial link filter: {e}", file=sys.stderr)
285
+
286
  filter_path = Path("filter.txt")
287
  if filter_path.exists():
288
  try:
 
303
  current_url, current_depth = next_url_row
304
  con.execute("DELETE FROM queue WHERE url = ?", (current_url,)) # Remove from queue
305
 
306
+ # Check against regex (skip if doesn't match).
307
+ # NOTE: start_url is always allowed through so the crawl can begin,
308
+ # but every other URL — including ones pre-queued before the filter
309
+ # was set — must match the regex.
310
  if filter_regex and current_url != start_url and not filter_regex.search(current_url):
311
  # print(f"Skipping {current_url} (regex mismatch)", flush=True)
312
  continue
 
521
  parser.add_argument("--max-depth", type=int, default=None, help="Maximum depth to scrape (default: unlimited)")
522
  parser.add_argument("--keep-pdfs", action="store_true", help="Keep PDFs in original format")
523
  parser.add_argument("--log-file", help="Path to log file")
524
+ parser.add_argument("--link-filter", help="Regex pattern to filter links")
525
 
526
  args = parser.parse_args()
527
 
 
542
  print("Error: Could not derive database name from URL. Please specify db_path.")
543
  sys.exit(1)
544
 
545
+ scrape(args.url, db_path, throttle_rate=args.throttle, max_reqs=args.max_reqs, max_depth=args.max_depth, keep_pdfs=args.keep_pdfs, use_cloudscraper=args.cloudscraper, use_selenium=args.selenium, use_sitemap=args.sitemap, link_filter=args.link_filter)