Add compression of db
Browse files- tools/scraper.py +2 -2
- tools/worker.py +54 -17
tools/scraper.py
CHANGED
|
@@ -531,7 +531,7 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 531 |
out_md = target_dir / (path.stem + "_combined.md")
|
| 532 |
|
| 533 |
try:
|
| 534 |
-
con = duckdb.connect(str(path), read_only=True)
|
| 535 |
pages = con.execute("SELECT url, title, content FROM markdown WHERE content IS NOT NULL AND content != '' ORDER BY url").fetchall()
|
| 536 |
con.close()
|
| 537 |
|
|
@@ -570,7 +570,7 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
|
|
| 570 |
temp_path = Path(temp_dir)
|
| 571 |
|
| 572 |
|
| 573 |
-
con = duckdb.connect(str(path))
|
| 574 |
try:
|
| 575 |
# Check link for markdown table
|
| 576 |
try:
|
|
|
|
| 531 |
out_md = target_dir / (path.stem + "_combined.md")
|
| 532 |
|
| 533 |
try:
|
| 534 |
+
con = duckdb.connect(str(path), read_only=True, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
|
| 535 |
pages = con.execute("SELECT url, title, content FROM markdown WHERE content IS NOT NULL AND content != '' ORDER BY url").fetchall()
|
| 536 |
con.close()
|
| 537 |
|
|
|
|
| 570 |
temp_path = Path(temp_dir)
|
| 571 |
|
| 572 |
|
| 573 |
+
con = duckdb.connect(str(path), config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
|
| 574 |
try:
|
| 575 |
# Check link for markdown table
|
| 576 |
try:
|
tools/worker.py
CHANGED
|
@@ -20,7 +20,7 @@ logging.getLogger("pdfplumber").setLevel(logging.ERROR)
|
|
| 20 |
logging.getLogger("pypdf").setLevel(logging.ERROR)
|
| 21 |
|
| 22 |
def normalize_url(url):
|
| 23 |
-
if
|
| 24 |
url = "https://" + url
|
| 25 |
|
| 26 |
# Strip fragment
|
|
@@ -34,10 +34,35 @@ def normalize_url(url):
|
|
| 34 |
def get_host(url):
|
| 35 |
try:
|
| 36 |
url = normalize_url(url)
|
| 37 |
-
|
|
|
|
| 38 |
except Exception:
|
| 39 |
return None
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
def make_req(url, throttle_rate=2, user_agent=None, cloudscraper_session=None):
|
| 42 |
if cloudscraper_session:
|
| 43 |
time.sleep(1.0 / throttle_rate)
|
|
@@ -84,7 +109,10 @@ def extract_links(html_content, current_url):
|
|
| 84 |
|
| 85 |
try:
|
| 86 |
absolute_url = urljoin(current_url, href)
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
| 88 |
except Exception:
|
| 89 |
continue
|
| 90 |
|
|
@@ -99,34 +127,34 @@ def init_db(con, page_table="pages", doc_table="documents", queue_table="queue")
|
|
| 99 |
con.execute(f"""
|
| 100 |
CREATE TABLE IF NOT EXISTS {page_table} (
|
| 101 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 102 |
-
url VARCHAR PRIMARY KEY,
|
| 103 |
-
html TEXT,
|
| 104 |
-
title VARCHAR
|
| 105 |
);
|
| 106 |
""")
|
| 107 |
con.execute(f"""
|
| 108 |
CREATE TABLE IF NOT EXISTS {doc_table} (
|
| 109 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 110 |
-
url VARCHAR PRIMARY KEY,
|
| 111 |
-
source_page VARCHAR,
|
| 112 |
size BIGINT
|
| 113 |
);
|
| 114 |
""")
|
| 115 |
con.execute(f"""
|
| 116 |
CREATE TABLE IF NOT EXISTS {queue_table} (
|
| 117 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 118 |
-
url VARCHAR PRIMARY KEY,
|
| 119 |
depth INTEGER DEFAULT 0
|
| 120 |
);
|
| 121 |
""")
|
| 122 |
con.execute(f"""
|
| 123 |
CREATE TABLE IF NOT EXISTS markdown (
|
| 124 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 125 |
-
url VARCHAR PRIMARY KEY,
|
| 126 |
-
title VARCHAR,
|
| 127 |
-
content TEXT,
|
| 128 |
-
source_type VARCHAR,
|
| 129 |
-
binary_content BLOB
|
| 130 |
);
|
| 131 |
""")
|
| 132 |
|
|
@@ -183,7 +211,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 183 |
print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
|
| 184 |
print(f"-----------------------", flush=True)
|
| 185 |
|
| 186 |
-
con = duckdb.connect(db_path)
|
| 187 |
init_db(con)
|
| 188 |
|
| 189 |
cloudscraper_session = None
|
|
@@ -221,8 +249,9 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 221 |
# Can use a regular request (or cloudscraper) since sitemaps are usually static XML
|
| 222 |
resp = make_req(start_url, throttle_rate, cloudscraper_session=cloudscraper_session)
|
| 223 |
if resp and resp.status_code == 200:
|
|
|
|
| 224 |
import warnings
|
| 225 |
-
from bs4 import
|
| 226 |
# parse xml using html.parser, finding <loc> works well enough
|
| 227 |
warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
|
| 228 |
sitemap_soup = BeautifulSoup(resp.content, 'html.parser')
|
|
@@ -310,6 +339,14 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 310 |
html = resp.text
|
| 311 |
resolved_url = resp.url
|
| 312 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
# Extract title
|
| 314 |
soup = BeautifulSoup(html, 'html.parser')
|
| 315 |
page_title = soup.title.string.strip() if soup.title and soup.title.string else None
|
|
@@ -346,7 +383,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 346 |
|
| 347 |
for link in web_links:
|
| 348 |
# Host filter
|
| 349 |
-
if get_host(link)
|
| 350 |
continue
|
| 351 |
|
| 352 |
# Duplicate filter
|
|
|
|
| 20 |
logging.getLogger("pypdf").setLevel(logging.ERROR)
|
| 21 |
|
| 22 |
def normalize_url(url):
|
| 23 |
+
if "://" not in url:
|
| 24 |
url = "https://" + url
|
| 25 |
|
| 26 |
# Strip fragment
|
|
|
|
| 34 |
def get_host(url):
|
| 35 |
try:
|
| 36 |
url = normalize_url(url)
|
| 37 |
+
host = urlparse(url).hostname
|
| 38 |
+
return host.lower() if host else None
|
| 39 |
except Exception:
|
| 40 |
return None
|
| 41 |
|
| 42 |
+
def is_same_site(host1, host2):
|
| 43 |
+
if not host1 or not host2:
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
h1 = host1.lower()
|
| 47 |
+
h2 = host2.lower()
|
| 48 |
+
|
| 49 |
+
if h1 == h2:
|
| 50 |
+
return True
|
| 51 |
+
|
| 52 |
+
# Handle www. and subdomains
|
| 53 |
+
h1_root = h1.removeprefix("www.")
|
| 54 |
+
h2_root = h2.lower().removeprefix("www.")
|
| 55 |
+
|
| 56 |
+
if h1_root == h2_root:
|
| 57 |
+
return True
|
| 58 |
+
|
| 59 |
+
# Allow subdomains: check if one is a suffix of the other with a dot
|
| 60 |
+
# e.g. news.forsvaret.no is part of forsvaret.no
|
| 61 |
+
if h1.endswith("." + h2_root) or h2.endswith("." + h1_root):
|
| 62 |
+
return True
|
| 63 |
+
|
| 64 |
+
return False
|
| 65 |
+
|
| 66 |
def make_req(url, throttle_rate=2, user_agent=None, cloudscraper_session=None):
|
| 67 |
if cloudscraper_session:
|
| 68 |
time.sleep(1.0 / throttle_rate)
|
|
|
|
| 109 |
|
| 110 |
try:
|
| 111 |
absolute_url = urljoin(current_url, href)
|
| 112 |
+
normalized = normalize_url(absolute_url)
|
| 113 |
+
# Only include http and https schemes
|
| 114 |
+
if urlparse(normalized).scheme in ['http', 'https']:
|
| 115 |
+
all_links.add(normalized)
|
| 116 |
except Exception:
|
| 117 |
continue
|
| 118 |
|
|
|
|
| 127 |
con.execute(f"""
|
| 128 |
CREATE TABLE IF NOT EXISTS {page_table} (
|
| 129 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 130 |
+
url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
|
| 131 |
+
html TEXT USING COMPRESSION 'zstd',
|
| 132 |
+
title VARCHAR USING COMPRESSION 'zstd'
|
| 133 |
);
|
| 134 |
""")
|
| 135 |
con.execute(f"""
|
| 136 |
CREATE TABLE IF NOT EXISTS {doc_table} (
|
| 137 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 138 |
+
url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
|
| 139 |
+
source_page VARCHAR USING COMPRESSION 'zstd',
|
| 140 |
size BIGINT
|
| 141 |
);
|
| 142 |
""")
|
| 143 |
con.execute(f"""
|
| 144 |
CREATE TABLE IF NOT EXISTS {queue_table} (
|
| 145 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 146 |
+
url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
|
| 147 |
depth INTEGER DEFAULT 0
|
| 148 |
);
|
| 149 |
""")
|
| 150 |
con.execute(f"""
|
| 151 |
CREATE TABLE IF NOT EXISTS markdown (
|
| 152 |
insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 153 |
+
url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
|
| 154 |
+
title VARCHAR USING COMPRESSION 'zstd',
|
| 155 |
+
content TEXT USING COMPRESSION 'zstd',
|
| 156 |
+
source_type VARCHAR USING COMPRESSION 'zstd',
|
| 157 |
+
binary_content BLOB USING COMPRESSION 'zstd'
|
| 158 |
);
|
| 159 |
""")
|
| 160 |
|
|
|
|
| 211 |
print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
|
| 212 |
print(f"-----------------------", flush=True)
|
| 213 |
|
| 214 |
+
con = duckdb.connect(db_path, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
|
| 215 |
init_db(con)
|
| 216 |
|
| 217 |
cloudscraper_session = None
|
|
|
|
| 249 |
# Can use a regular request (or cloudscraper) since sitemaps are usually static XML
|
| 250 |
resp = make_req(start_url, throttle_rate, cloudscraper_session=cloudscraper_session)
|
| 251 |
if resp and resp.status_code == 200:
|
| 252 |
+
# parse xml using html.parser, finding <loc> works well enough
|
| 253 |
import warnings
|
| 254 |
+
from bs4 import XMLParsedAsHTMLWarning
|
| 255 |
# parse xml using html.parser, finding <loc> works well enough
|
| 256 |
warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
|
| 257 |
sitemap_soup = BeautifulSoup(resp.content, 'html.parser')
|
|
|
|
| 339 |
html = resp.text
|
| 340 |
resolved_url = resp.url
|
| 341 |
|
| 342 |
+
# Update base_host if this is the first successful request
|
| 343 |
+
# This handles redirects from root to www or vice versa
|
| 344 |
+
if reqs_made == 1:
|
| 345 |
+
new_host = get_host(resolved_url)
|
| 346 |
+
if new_host and not is_same_site(new_host, base_host):
|
| 347 |
+
print(f"Updating base_host to {new_host} (redirect from {base_host})", flush=True)
|
| 348 |
+
base_host = new_host
|
| 349 |
+
|
| 350 |
# Extract title
|
| 351 |
soup = BeautifulSoup(html, 'html.parser')
|
| 352 |
page_title = soup.title.string.strip() if soup.title and soup.title.string else None
|
|
|
|
| 383 |
|
| 384 |
for link in web_links:
|
| 385 |
# Host filter
|
| 386 |
+
if not is_same_site(get_host(link), base_host):
|
| 387 |
continue
|
| 388 |
|
| 389 |
# Duplicate filter
|