thohan88 commited on
Commit
267e34a
·
1 Parent(s): 5f30904

Add compression of db

Browse files
Files changed (2) hide show
  1. tools/scraper.py +2 -2
  2. tools/worker.py +54 -17
tools/scraper.py CHANGED
@@ -531,7 +531,7 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
531
  out_md = target_dir / (path.stem + "_combined.md")
532
 
533
  try:
534
- con = duckdb.connect(str(path), read_only=True)
535
  pages = con.execute("SELECT url, title, content FROM markdown WHERE content IS NOT NULL AND content != '' ORDER BY url").fetchall()
536
  con.close()
537
 
@@ -570,7 +570,7 @@ def mod_scraper_server(input, output, session, www_dir, scrapes_dir):
570
  temp_path = Path(temp_dir)
571
 
572
 
573
- con = duckdb.connect(str(path))
574
  try:
575
  # Check link for markdown table
576
  try:
 
531
  out_md = target_dir / (path.stem + "_combined.md")
532
 
533
  try:
534
+ con = duckdb.connect(str(path), read_only=True, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
535
  pages = con.execute("SELECT url, title, content FROM markdown WHERE content IS NOT NULL AND content != '' ORDER BY url").fetchall()
536
  con.close()
537
 
 
570
  temp_path = Path(temp_dir)
571
 
572
 
573
+ con = duckdb.connect(str(path), config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
574
  try:
575
  # Check link for markdown table
576
  try:
tools/worker.py CHANGED
@@ -20,7 +20,7 @@ logging.getLogger("pdfplumber").setLevel(logging.ERROR)
20
  logging.getLogger("pypdf").setLevel(logging.ERROR)
21
 
22
  def normalize_url(url):
23
- if not url.startswith("http://") and not url.startswith("https://"):
24
  url = "https://" + url
25
 
26
  # Strip fragment
@@ -34,10 +34,35 @@ def normalize_url(url):
34
  def get_host(url):
35
  try:
36
  url = normalize_url(url)
37
- return urlparse(url).hostname
 
38
  except Exception:
39
  return None
40
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  def make_req(url, throttle_rate=2, user_agent=None, cloudscraper_session=None):
42
  if cloudscraper_session:
43
  time.sleep(1.0 / throttle_rate)
@@ -84,7 +109,10 @@ def extract_links(html_content, current_url):
84
 
85
  try:
86
  absolute_url = urljoin(current_url, href)
87
- all_links.add(normalize_url(absolute_url))
 
 
 
88
  except Exception:
89
  continue
90
 
@@ -99,34 +127,34 @@ def init_db(con, page_table="pages", doc_table="documents", queue_table="queue")
99
  con.execute(f"""
100
  CREATE TABLE IF NOT EXISTS {page_table} (
101
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
102
- url VARCHAR PRIMARY KEY,
103
- html TEXT,
104
- title VARCHAR
105
  );
106
  """)
107
  con.execute(f"""
108
  CREATE TABLE IF NOT EXISTS {doc_table} (
109
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
110
- url VARCHAR PRIMARY KEY,
111
- source_page VARCHAR,
112
  size BIGINT
113
  );
114
  """)
115
  con.execute(f"""
116
  CREATE TABLE IF NOT EXISTS {queue_table} (
117
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
118
- url VARCHAR PRIMARY KEY,
119
  depth INTEGER DEFAULT 0
120
  );
121
  """)
122
  con.execute(f"""
123
  CREATE TABLE IF NOT EXISTS markdown (
124
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
125
- url VARCHAR PRIMARY KEY,
126
- title VARCHAR,
127
- content TEXT,
128
- source_type VARCHAR,
129
- binary_content BLOB
130
  );
131
  """)
132
 
@@ -183,7 +211,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
183
  print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
184
  print(f"-----------------------", flush=True)
185
 
186
- con = duckdb.connect(db_path)
187
  init_db(con)
188
 
189
  cloudscraper_session = None
@@ -221,8 +249,9 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
221
  # Can use a regular request (or cloudscraper) since sitemaps are usually static XML
222
  resp = make_req(start_url, throttle_rate, cloudscraper_session=cloudscraper_session)
223
  if resp and resp.status_code == 200:
 
224
  import warnings
225
- from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
226
  # parse xml using html.parser, finding <loc> works well enough
227
  warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
228
  sitemap_soup = BeautifulSoup(resp.content, 'html.parser')
@@ -310,6 +339,14 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
310
  html = resp.text
311
  resolved_url = resp.url
312
 
 
 
 
 
 
 
 
 
313
  # Extract title
314
  soup = BeautifulSoup(html, 'html.parser')
315
  page_title = soup.title.string.strip() if soup.title and soup.title.string else None
@@ -346,7 +383,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
346
 
347
  for link in web_links:
348
  # Host filter
349
- if get_host(link) != base_host:
350
  continue
351
 
352
  # Duplicate filter
 
20
  logging.getLogger("pypdf").setLevel(logging.ERROR)
21
 
22
  def normalize_url(url):
23
+ if "://" not in url:
24
  url = "https://" + url
25
 
26
  # Strip fragment
 
34
  def get_host(url):
35
  try:
36
  url = normalize_url(url)
37
+ host = urlparse(url).hostname
38
+ return host.lower() if host else None
39
  except Exception:
40
  return None
41
 
42
+ def is_same_site(host1, host2):
43
+ if not host1 or not host2:
44
+ return False
45
+
46
+ h1 = host1.lower()
47
+ h2 = host2.lower()
48
+
49
+ if h1 == h2:
50
+ return True
51
+
52
+ # Handle www. and subdomains
53
+ h1_root = h1.removeprefix("www.")
54
+ h2_root = h2.lower().removeprefix("www.")
55
+
56
+ if h1_root == h2_root:
57
+ return True
58
+
59
+ # Allow subdomains: check if one is a suffix of the other with a dot
60
+ # e.g. news.forsvaret.no is part of forsvaret.no
61
+ if h1.endswith("." + h2_root) or h2.endswith("." + h1_root):
62
+ return True
63
+
64
+ return False
65
+
66
  def make_req(url, throttle_rate=2, user_agent=None, cloudscraper_session=None):
67
  if cloudscraper_session:
68
  time.sleep(1.0 / throttle_rate)
 
109
 
110
  try:
111
  absolute_url = urljoin(current_url, href)
112
+ normalized = normalize_url(absolute_url)
113
+ # Only include http and https schemes
114
+ if urlparse(normalized).scheme in ['http', 'https']:
115
+ all_links.add(normalized)
116
  except Exception:
117
  continue
118
 
 
127
  con.execute(f"""
128
  CREATE TABLE IF NOT EXISTS {page_table} (
129
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
130
+ url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
131
+ html TEXT USING COMPRESSION 'zstd',
132
+ title VARCHAR USING COMPRESSION 'zstd'
133
  );
134
  """)
135
  con.execute(f"""
136
  CREATE TABLE IF NOT EXISTS {doc_table} (
137
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
138
+ url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
139
+ source_page VARCHAR USING COMPRESSION 'zstd',
140
  size BIGINT
141
  );
142
  """)
143
  con.execute(f"""
144
  CREATE TABLE IF NOT EXISTS {queue_table} (
145
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
146
+ url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
147
  depth INTEGER DEFAULT 0
148
  );
149
  """)
150
  con.execute(f"""
151
  CREATE TABLE IF NOT EXISTS markdown (
152
  insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
153
+ url VARCHAR USING COMPRESSION 'zstd' PRIMARY KEY,
154
+ title VARCHAR USING COMPRESSION 'zstd',
155
+ content TEXT USING COMPRESSION 'zstd',
156
+ source_type VARCHAR USING COMPRESSION 'zstd',
157
+ binary_content BLOB USING COMPRESSION 'zstd'
158
  );
159
  """)
160
 
 
211
  print(f"Features: {opts_str} | Max Depth: {depth_str} | Max Reqs: {max_reqs} | Keep PDFs: {keep_pdfs}", flush=True)
212
  print(f"-----------------------", flush=True)
213
 
214
+ con = duckdb.connect(db_path, config={'storage_compatibility_version': 'latest', 'zstd_min_string_length': 1024})
215
  init_db(con)
216
 
217
  cloudscraper_session = None
 
249
  # Can use a regular request (or cloudscraper) since sitemaps are usually static XML
250
  resp = make_req(start_url, throttle_rate, cloudscraper_session=cloudscraper_session)
251
  if resp and resp.status_code == 200:
252
+ # parse xml using html.parser, finding <loc> works well enough
253
  import warnings
254
+ from bs4 import XMLParsedAsHTMLWarning
255
  # parse xml using html.parser, finding <loc> works well enough
256
  warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
257
  sitemap_soup = BeautifulSoup(resp.content, 'html.parser')
 
339
  html = resp.text
340
  resolved_url = resp.url
341
 
342
+ # Update base_host if this is the first successful request
343
+ # This handles redirects from root to www or vice versa
344
+ if reqs_made == 1:
345
+ new_host = get_host(resolved_url)
346
+ if new_host and not is_same_site(new_host, base_host):
347
+ print(f"Updating base_host to {new_host} (redirect from {base_host})", flush=True)
348
+ base_host = new_host
349
+
350
  # Extract title
351
  soup = BeautifulSoup(html, 'html.parser')
352
  page_title = soup.title.string.strip() if soup.title and soup.title.string else None
 
383
 
384
  for link in web_links:
385
  # Host filter
386
+ if not is_same_site(get_host(link), base_host):
387
  continue
388
 
389
  # Duplicate filter