feat: add support for path-based link filtering and mutual exclusion for scraper modes
Browse files- tools/scraper.py +49 -7
- tools/worker.py +24 -15
tools/scraper.py
CHANGED
|
@@ -97,10 +97,19 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 97 |
current_filter = input.link_filter()
|
| 98 |
last_auto = last_auto_filter.get()
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
# Update if currently empty OR if it matches what we last auto-set
|
| 101 |
if not current_filter or current_filter == last_auto:
|
| 102 |
-
ui.update_text("link_filter", value=
|
| 103 |
-
last_auto_filter.set(
|
| 104 |
|
| 105 |
@reactive.effect
|
| 106 |
def auto_check_sitemap():
|
|
@@ -108,15 +117,39 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 108 |
if url and url.strip().lower().endswith("sitemap.xml"):
|
| 109 |
ui.update_checkbox("sitemap", value=True, session=session)
|
| 110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
@reactive.effect
|
| 112 |
def update_filter():
|
| 113 |
-
path = scrapes_dir / "filter.txt"
|
| 114 |
regex = input.link_filter()
|
|
|
|
|
|
|
|
|
|
| 115 |
if regex:
|
| 116 |
with open(path, "w") as f:
|
| 117 |
f.write(regex)
|
| 118 |
elif path.exists():
|
| 119 |
path.unlink()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
@reactive.poll(check_log_mtime, interval_secs=0.1)
|
| 122 |
def log_content():
|
|
@@ -210,6 +243,9 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 210 |
scraper_script = Path(__file__).parent / "worker.py"
|
| 211 |
|
| 212 |
cmd = [sys.executable, "-u", str(scraper_script), url, str(db_path), "--log-file", str(log_path)]
|
|
|
|
|
|
|
|
|
|
| 213 |
|
| 214 |
|
| 215 |
if input.cloudscraper():
|
|
@@ -504,8 +540,8 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 504 |
),
|
| 505 |
ui.div(
|
| 506 |
ui.download_button("download_db", "Download Database", class_="btn-success"),
|
| 507 |
-
ui.download_button("download_md", "Download Markdown", class_="btn-info"),
|
| 508 |
-
ui.download_button("download_combined_md", "Download Combined
|
| 509 |
class_="d-flex gap-2 justify-content-center"
|
| 510 |
),
|
| 511 |
class_="d-flex flex-column align-items-center"
|
|
@@ -518,7 +554,10 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 518 |
if path and path.exists():
|
| 519 |
return str(path)
|
| 520 |
|
| 521 |
-
@render.download(
|
|
|
|
|
|
|
|
|
|
| 522 |
def download_combined_md():
|
| 523 |
path = db_file_path.get()
|
| 524 |
if not path or not path.exists():
|
|
@@ -552,7 +591,10 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 552 |
print(f"Error generating combined markdown: {e}")
|
| 553 |
return None
|
| 554 |
|
| 555 |
-
@render.download(
|
|
|
|
|
|
|
|
|
|
| 556 |
def download_md():
|
| 557 |
path = db_file_path.get()
|
| 558 |
if not path or not path.exists():
|
|
|
|
| 97 |
current_filter = input.link_filter()
|
| 98 |
last_auto = last_auto_filter.get()
|
| 99 |
|
| 100 |
+
# Build filter as host + path (strip trailing slash) so that
|
| 101 |
+
# e.g. https://developers.cloudflare.com/containers/ auto-fills
|
| 102 |
+
# "developers.cloudflare.com/containers" not just the hostname.
|
| 103 |
+
try:
|
| 104 |
+
parsed_path = urlparse(url).path.rstrip("/")
|
| 105 |
+
except Exception:
|
| 106 |
+
parsed_path = ""
|
| 107 |
+
auto_filter = host + parsed_path if parsed_path and parsed_path != "/" else host
|
| 108 |
+
|
| 109 |
# Update if currently empty OR if it matches what we last auto-set
|
| 110 |
if not current_filter or current_filter == last_auto:
|
| 111 |
+
ui.update_text("link_filter", value=auto_filter, session=session)
|
| 112 |
+
last_auto_filter.set(auto_filter)
|
| 113 |
|
| 114 |
@reactive.effect
|
| 115 |
def auto_check_sitemap():
|
|
|
|
| 117 |
if url and url.strip().lower().endswith("sitemap.xml"):
|
| 118 |
ui.update_checkbox("sitemap", value=True, session=session)
|
| 119 |
|
| 120 |
+
@reactive.effect
|
| 121 |
+
@reactive.event(input.cloudscraper)
|
| 122 |
+
def _():
|
| 123 |
+
if input.cloudscraper():
|
| 124 |
+
ui.update_checkbox("selenium", value=False, session=session)
|
| 125 |
+
|
| 126 |
+
@reactive.effect
|
| 127 |
+
@reactive.event(input.selenium)
|
| 128 |
+
def _():
|
| 129 |
+
if input.selenium():
|
| 130 |
+
ui.update_checkbox("cloudscraper", value=False, session=session)
|
| 131 |
+
|
| 132 |
@reactive.effect
|
| 133 |
def update_filter():
|
|
|
|
| 134 |
regex = input.link_filter()
|
| 135 |
+
|
| 136 |
+
# Update global filter (for new jobs)
|
| 137 |
+
path = scrapes_dir / "filter.txt"
|
| 138 |
if regex:
|
| 139 |
with open(path, "w") as f:
|
| 140 |
f.write(regex)
|
| 141 |
elif path.exists():
|
| 142 |
path.unlink()
|
| 143 |
+
|
| 144 |
+
# Also update current job's filter if one is active
|
| 145 |
+
db_path = db_file_path.get()
|
| 146 |
+
if db_path:
|
| 147 |
+
job_filter_path = db_path.parent / "filter.txt"
|
| 148 |
+
if regex:
|
| 149 |
+
with open(job_filter_path, "w") as f:
|
| 150 |
+
f.write(regex)
|
| 151 |
+
elif job_filter_path.exists():
|
| 152 |
+
job_filter_path.unlink()
|
| 153 |
|
| 154 |
@reactive.poll(check_log_mtime, interval_secs=0.1)
|
| 155 |
def log_content():
|
|
|
|
| 243 |
scraper_script = Path(__file__).parent / "worker.py"
|
| 244 |
|
| 245 |
cmd = [sys.executable, "-u", str(scraper_script), url, str(db_path), "--log-file", str(log_path)]
|
| 246 |
+
|
| 247 |
+
if input.link_filter():
|
| 248 |
+
cmd.extend(["--link-filter", input.link_filter()])
|
| 249 |
|
| 250 |
|
| 251 |
if input.cloudscraper():
|
|
|
|
| 540 |
),
|
| 541 |
ui.div(
|
| 542 |
ui.download_button("download_db", "Download Database", class_="btn-success"),
|
| 543 |
+
ui.download_button("download_md", "Download Markdown (ZIP)", class_="btn-info"),
|
| 544 |
+
ui.download_button("download_combined_md", "Download Combined Markdown", class_="btn-primary"),
|
| 545 |
class_="d-flex gap-2 justify-content-center"
|
| 546 |
),
|
| 547 |
class_="d-flex flex-column align-items-center"
|
|
|
|
| 554 |
if path and path.exists():
|
| 555 |
return str(path)
|
| 556 |
|
| 557 |
+
@render.download(
|
| 558 |
+
filename=lambda: f"{Path(db_file_path.get()).stem}_combined.md" if db_file_path.get() else "combined.md",
|
| 559 |
+
media_type="text/markdown"
|
| 560 |
+
)
|
| 561 |
def download_combined_md():
|
| 562 |
path = db_file_path.get()
|
| 563 |
if not path or not path.exists():
|
|
|
|
| 591 |
print(f"Error generating combined markdown: {e}")
|
| 592 |
return None
|
| 593 |
|
| 594 |
+
@render.download(
|
| 595 |
+
filename=lambda: f"{Path(db_file_path.get()).stem}_markdown.zip" if db_file_path.get() else "markdown.zip",
|
| 596 |
+
media_type="application/zip"
|
| 597 |
+
)
|
| 598 |
def download_md():
|
| 599 |
path = db_file_path.get()
|
| 600 |
if not path or not path.exists():
|
tools/worker.py
CHANGED
|
@@ -7,9 +7,9 @@ from bs4 import BeautifulSoup
|
|
| 7 |
import os
|
| 8 |
import argparse
|
| 9 |
import sys
|
| 10 |
-
import re
|
| 11 |
from pathlib import Path
|
| 12 |
import tempfile
|
|
|
|
| 13 |
from markitdown import MarkItDown
|
| 14 |
import logging
|
| 15 |
import cloudscraper
|
|
@@ -40,6 +40,11 @@ def get_host(url):
|
|
| 40 |
return None
|
| 41 |
|
| 42 |
def is_same_site(host1, host2):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
if not host1 or not host2:
|
| 44 |
return False
|
| 45 |
|
|
@@ -49,16 +54,8 @@ def is_same_site(host1, host2):
|
|
| 49 |
if h1 == h2:
|
| 50 |
return True
|
| 51 |
|
| 52 |
-
#
|
| 53 |
-
|
| 54 |
-
h2_root = h2.lower().removeprefix("www.")
|
| 55 |
-
|
| 56 |
-
if h1_root == h2_root:
|
| 57 |
-
return True
|
| 58 |
-
|
| 59 |
-
# Allow subdomains: check if one is a suffix of the other with a dot
|
| 60 |
-
# e.g. news.forsvaret.no is part of forsvaret.no
|
| 61 |
-
if h1.endswith("." + h2_root) or h2.endswith("." + h1_root):
|
| 62 |
return True
|
| 63 |
|
| 64 |
return False
|
|
@@ -193,7 +190,7 @@ def store_documents(con, doc_links, source_url, doc_table="documents"):
|
|
| 193 |
con.execute(f"INSERT INTO {doc_table} (url, source_page, size) VALUES (?, ?, ?)",
|
| 194 |
(link, source_url, size))
|
| 195 |
|
| 196 |
-
def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, keep_pdfs=False, use_cloudscraper=False, use_selenium=False, use_sitemap=False):
|
| 197 |
start_url = normalize_url(start_url)
|
| 198 |
base_host = get_host(start_url)
|
| 199 |
|
|
@@ -209,11 +206,14 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 209 |
print(f"--- Scraper Started ---", flush=True)
|
| 210 |
print(f"Target: {start_url}", flush=True)
|
| 211 |
print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
|
|
|
|
|
|
|
| 212 |
print(f"-----------------------", flush=True)
|
| 213 |
|
| 214 |
con = duckdb.connect(db_path, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
|
| 215 |
init_db(con)
|
| 216 |
|
|
|
|
| 217 |
cloudscraper_session = None
|
| 218 |
if use_cloudscraper:
|
| 219 |
try:
|
|
@@ -277,6 +277,12 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 277 |
while reqs_made < max_reqs:
|
| 278 |
# Dynamic regex filter
|
| 279 |
filter_regex = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
filter_path = Path("filter.txt")
|
| 281 |
if filter_path.exists():
|
| 282 |
try:
|
|
@@ -297,8 +303,10 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 297 |
current_url, current_depth = next_url_row
|
| 298 |
con.execute("DELETE FROM queue WHERE url = ?", (current_url,)) # Remove from queue
|
| 299 |
|
| 300 |
-
# Check against regex (skip if doesn't match)
|
| 301 |
-
#
|
|
|
|
|
|
|
| 302 |
if filter_regex and current_url != start_url and not filter_regex.search(current_url):
|
| 303 |
# print(f"Skipping {current_url} (regex mismatch)", flush=True)
|
| 304 |
continue
|
|
@@ -513,6 +521,7 @@ if __name__ == "__main__":
|
|
| 513 |
parser.add_argument("--max-depth", type=int, default=None, help="Maximum depth to scrape (default: unlimited)")
|
| 514 |
parser.add_argument("--keep-pdfs", action="store_true", help="Keep PDFs in original format")
|
| 515 |
parser.add_argument("--log-file", help="Path to log file")
|
|
|
|
| 516 |
|
| 517 |
args = parser.parse_args()
|
| 518 |
|
|
@@ -533,4 +542,4 @@ if __name__ == "__main__":
|
|
| 533 |
print("Error: Could not derive database name from URL. Please specify db_path.")
|
| 534 |
sys.exit(1)
|
| 535 |
|
| 536 |
-
scrape(args.url, db_path, throttle_rate=args.throttle, max_reqs=args.max_reqs, max_depth=args.max_depth, keep_pdfs=args.keep_pdfs, use_cloudscraper=args.cloudscraper, use_selenium=args.selenium, use_sitemap=args.sitemap)
|
|
|
|
| 7 |
import os
|
| 8 |
import argparse
|
| 9 |
import sys
|
|
|
|
| 10 |
from pathlib import Path
|
| 11 |
import tempfile
|
| 12 |
+
import hashlib
|
| 13 |
from markitdown import MarkItDown
|
| 14 |
import logging
|
| 15 |
import cloudscraper
|
|
|
|
| 40 |
return None
|
| 41 |
|
| 42 |
def is_same_site(host1, host2):
|
| 43 |
+
"""Return True only if the two hosts are the exact same host,
|
| 44 |
+
or one is the bare domain and the other is the www. variant.
|
| 45 |
+
Subdomains (e.g. news.example.com vs example.com) are intentionally
|
| 46 |
+
NOT treated as same-site, because that causes cross-subdomain leakage
|
| 47 |
+
when a path-based link_filter is in use."""
|
| 48 |
if not host1 or not host2:
|
| 49 |
return False
|
| 50 |
|
|
|
|
| 54 |
if h1 == h2:
|
| 55 |
return True
|
| 56 |
|
| 57 |
+
# Allow only the www. <-> bare-domain pair, nothing else
|
| 58 |
+
if h1.removeprefix("www.") == h2.removeprefix("www."):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
return True
|
| 60 |
|
| 61 |
return False
|
|
|
|
| 190 |
con.execute(f"INSERT INTO {doc_table} (url, source_page, size) VALUES (?, ?, ?)",
|
| 191 |
(link, source_url, size))
|
| 192 |
|
| 193 |
+
def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, keep_pdfs=False, use_cloudscraper=False, use_selenium=False, use_sitemap=False, link_filter=None):
|
| 194 |
start_url = normalize_url(start_url)
|
| 195 |
base_host = get_host(start_url)
|
| 196 |
|
|
|
|
| 206 |
print(f"--- Scraper Started ---", flush=True)
|
| 207 |
print(f"Target: {start_url}", flush=True)
|
| 208 |
print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
|
| 209 |
+
if link_filter:
|
| 210 |
+
print(f"Link Filter: {link_filter}", flush=True)
|
| 211 |
print(f"-----------------------", flush=True)
|
| 212 |
|
| 213 |
con = duckdb.connect(db_path, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
|
| 214 |
init_db(con)
|
| 215 |
|
| 216 |
+
|
| 217 |
cloudscraper_session = None
|
| 218 |
if use_cloudscraper:
|
| 219 |
try:
|
|
|
|
| 277 |
while reqs_made < max_reqs:
|
| 278 |
# Dynamic regex filter
|
| 279 |
filter_regex = None
|
| 280 |
+
if link_filter:
|
| 281 |
+
try:
|
| 282 |
+
filter_regex = re.compile(link_filter)
|
| 283 |
+
except Exception as e:
|
| 284 |
+
print(f"Error compiling initial link filter: {e}", file=sys.stderr)
|
| 285 |
+
|
| 286 |
filter_path = Path("filter.txt")
|
| 287 |
if filter_path.exists():
|
| 288 |
try:
|
|
|
|
| 303 |
current_url, current_depth = next_url_row
|
| 304 |
con.execute("DELETE FROM queue WHERE url = ?", (current_url,)) # Remove from queue
|
| 305 |
|
| 306 |
+
# Check against regex (skip if doesn't match).
|
| 307 |
+
# NOTE: start_url is always allowed through so the crawl can begin,
|
| 308 |
+
# but every other URL — including ones pre-queued before the filter
|
| 309 |
+
# was set — must match the regex.
|
| 310 |
if filter_regex and current_url != start_url and not filter_regex.search(current_url):
|
| 311 |
# print(f"Skipping {current_url} (regex mismatch)", flush=True)
|
| 312 |
continue
|
|
|
|
| 521 |
parser.add_argument("--max-depth", type=int, default=None, help="Maximum depth to scrape (default: unlimited)")
|
| 522 |
parser.add_argument("--keep-pdfs", action="store_true", help="Keep PDFs in original format")
|
| 523 |
parser.add_argument("--log-file", help="Path to log file")
|
| 524 |
+
parser.add_argument("--link-filter", help="Regex pattern to filter links")
|
| 525 |
|
| 526 |
args = parser.parse_args()
|
| 527 |
|
|
|
|
| 542 |
print("Error: Could not derive database name from URL. Please specify db_path.")
|
| 543 |
sys.exit(1)
|
| 544 |
|
| 545 |
+
scrape(args.url, db_path, throttle_rate=args.throttle, max_reqs=args.max_reqs, max_depth=args.max_depth, keep_pdfs=args.keep_pdfs, use_cloudscraper=args.cloudscraper, use_selenium=args.selenium, use_sitemap=args.sitemap, link_filter=args.link_filter)
|