ogx786 commited on
Commit
36b40b7
·
verified ·
1 Parent(s): 2217911

Update crawlerror

Browse files
Files changed (1) hide show
  1. crawlerror +30 -431
crawlerror CHANGED
@@ -1,431 +1,30 @@
1
- """
2
- HBL PDF crawler + extractor (v2)
3
- ----------------------------------
4
- Changes from v1:
5
- - Seeds every {function}/Downloads.aspx?library={lib}&site=/{function} combo
6
- directly, since MyWorkspace.aspx does not link out to other departments.
7
- - Handles SharePoint's "load more" postback (only ~67 items load per initial
8
- GET; the rest need repeated postback clicks).
9
- - Everything else (PDF download via cookie-cloned session, pdfplumber
10
- extraction, OCR fallback, JSON output) is unchanged from v1.
11
-
12
- Run this on an internet-connected machine, NOT the air-gapped server.
13
- Copy the output folder over afterward, same as your other offline transfers.
14
- """
15
-
16
- import os
17
- import re
18
- import json
19
- import time
20
- import hashlib
21
- from pathlib import Path
22
- from urllib.parse import urljoin, urlparse
23
-
24
- import requests
25
- from bs4 import BeautifulSoup
26
- from tqdm import tqdm
27
-
28
- import pdfplumber
29
-
30
- from selenium import webdriver
31
- from selenium.webdriver.edge.options import Options as EdgeOptions
32
- from selenium.webdriver.common.by import By
33
- from selenium.common.exceptions import WebDriverException
34
-
35
- try:
36
- from requests_negotiate_sspi import HttpNegotiateAuth
37
- HAS_SSPI = True
38
- except ImportError:
39
- HAS_SSPI = False
40
-
41
- # Optional OCR fallback — only imported if a scanned PDF is actually encountered
42
- def _ocr_pdf(pdf_path):
43
- import pytesseract
44
- from pdf2image import convert_from_path
45
- images = convert_from_path(pdf_path)
46
- text = ""
47
- for i, image in enumerate(images):
48
- text += f"\n--- OCR page {i+1} ---\n"
49
- text += pytesseract.image_to_string(image)
50
- return text
51
-
52
- # ---------------------------------------------------------------------------
53
- # Config
54
- # ---------------------------------------------------------------------------
55
- BASE_DOMAIN = "iamhbl.com"
56
- OUTPUT_DIR = Path("./hbl_pdf_crawl")
57
- PDF_DIR = OUTPUT_DIR / "pdfs"
58
- MAX_PAGES_TO_CRAWL = 3000
59
- REQUEST_DELAY_SECONDS = 0.5
60
- PAGE_LOAD_WAIT_SECONDS = 1.5 # let JS-rendered content finish loading before reading the page
61
- POSTBACK_WAIT_SECONDS = 1.2 # wait after each load-more click for postback to settle
62
- MAX_LOAD_MORE_CLICKS = 60 # safety cap per page
63
- REQUEST_TIMEOUT = 20
64
-
65
- # ---------------------------------------------------------------------------
66
- # Seed matrix — every department function x library combo.
67
- # CONFIRM library spelling per department before running at scale (Circular vs
68
- # Circulars, Forms vs Form, etc. — SharePoint libraries are not guaranteed to
69
- # be named consistently). Open one Downloads.aspx per function in a browser
70
- # and read the tab hrefs to get the exact library= value.
71
- # ---------------------------------------------------------------------------
72
- FUNCTIONS = [
73
- "HOD", "CASB", "GCG", "ISE", "RB", "RM", "GT",
74
- "BOAS", "CIB", "CS", "DF", "EPT", "FCB", "FIGRB",
75
- "IA", "IB", "IBAN", "IFI", "ITSE", "LL", "MB", "OS",
76
- "RSC", "SQ", "SS", "TTPO",
77
- ] # from the department dropdown seen in Downloads.aspx; trim/extend as confirmed
78
-
79
- LIBRARIES = ["Circulars", "Forms", "FAQ"] # confirmed spelling from run output
80
-
81
- MAX_NETWORK_RETRIES = 4
82
- NETWORK_RETRY_BASE_DELAY = 5 # seconds; doubles each retry
83
-
84
- SEED_URLS = [
85
- f"https://iamhbl.com/{fn}/Pages/Downloads.aspx?library={lib}&site=%2f{fn}"
86
- for fn in FUNCTIONS
87
- for lib in LIBRARIES
88
- ]
89
-
90
- # ---------------------------------------------------------------------------
91
- # AUTH — attaches to an Edge window YOU already logged into manually, instead
92
- # of copying cookies by hand. Before running this script:
93
- # 1. Close every Edge window (check Task Manager for msedge.exe).
94
- # 2. Run: msedge.exe --remote-debugging-port=9222 --user-data-dir="C:\edge-debug-profile"
95
- # 3. Log into iamhbl.com in that window.
96
- # 4. Then run this script — it attaches to that same window and session.
97
- # ---------------------------------------------------------------------------
98
- EDGE_DEBUGGER_ADDRESS = "127.0.0.1:9222"
99
-
100
- # If Selenium's automatic driver download hangs or fails on this network-restricted
101
- # laptop, download msedgedriver.exe manually (matching your Edge version) from
102
- # https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/ and set
103
- # its path here. Leave as None to let Selenium try to auto-manage it.
104
- MSEDGEDRIVER_PATH = None # e.g. r"C:\edgedriver\msedgedriver.exe"
105
-
106
- def get_driver():
107
- options = EdgeOptions()
108
- options.debugger_address = EDGE_DEBUGGER_ADDRESS
109
- options.page_load_strategy = "eager" # return once DOM is ready, don't wait for every image/tracker to finish
110
- if MSEDGEDRIVER_PATH:
111
- from selenium.webdriver.edge.service import Service
112
- service = Service(executable_path=MSEDGEDRIVER_PATH)
113
- return webdriver.Edge(service=service, options=options)
114
- return webdriver.Edge(options=options)
115
-
116
- def looks_like_login_page(html_lower):
117
- return any(marker in html_lower for marker in
118
- ('id="username"', 'id="password"', 'sign in', 'log in to continue'))
119
-
120
- def looks_like_error_or_empty(html_lower):
121
- # SharePoint returns a rendered page even for a bad library= value, but the
122
- # repeater is empty. Catches the common "no items" / webpart error markers.
123
- return any(marker in html_lower for marker in
124
- ('an unexpected error has occurred', 'this list is empty', 'webpartzone1'))
125
-
126
- # ---------------------------------------------------------------------------
127
- # Load-more postback handling
128
- # ---------------------------------------------------------------------------
129
- LOAD_MORE_CANDIDATE_STRATEGIES = [
130
- # (By, selector) pairs tried in order; first visible match wins.
131
- (By.XPATH, "//a[contains(translate(text(),'LOADMORE','loadmore'),'load more')]"),
132
- (By.XPATH, "//button[contains(translate(text(),'LOADMORE','loadmore'),'load more')]"),
133
- (By.XPATH, "//*[contains(@id,'loadmore') or contains(@id,'LoadMore')]"),
134
- (By.CSS_SELECTOR, "[id*='loadmore' i]"),
135
- ]
136
-
137
- def find_load_more_button(driver):
138
- for by, selector in LOAD_MORE_CANDIDATE_STRATEGIES:
139
- try:
140
- elements = driver.find_elements(by, selector)
141
- for el in elements:
142
- if el.is_displayed() and el.is_enabled():
143
- return el
144
- except Exception:
145
- continue
146
- return None
147
-
148
- def click_load_more_until_done(driver, max_clicks=MAX_LOAD_MORE_CLICKS):
149
- """Repeatedly clicks the load-more control until it disappears or the page
150
- stops growing. Returns the number of clicks performed."""
151
- clicks = 0
152
- prev_html_len = len(driver.page_source)
153
- for _ in range(max_clicks):
154
- btn = find_load_more_button(driver)
155
- if btn is None:
156
- break
157
- try:
158
- driver.execute_script("arguments[0].scrollIntoView(true);", btn)
159
- btn.click()
160
- except Exception as e:
161
- print(f" [load-more click failed] {e}")
162
- break
163
- time.sleep(POSTBACK_WAIT_SECONDS)
164
- clicks += 1
165
- new_html_len = len(driver.page_source)
166
- if new_html_len <= prev_html_len:
167
- # postback happened but content didn't grow — stop to avoid a loop
168
- break
169
- prev_html_len = new_html_len
170
- return clicks
171
-
172
- # ---------------------------------------------------------------------------
173
- # Crawl: find every PDF URL, driving the real logged-in browser
174
- # ---------------------------------------------------------------------------
175
- def is_same_domain(url):
176
- netloc = urlparse(url).netloc.lower()
177
- return netloc == BASE_DOMAIN or netloc.endswith("." + BASE_DOMAIN)
178
-
179
- def safe_driver_get(driver, url, max_retries=MAX_NETWORK_RETRIES):
180
- """driver.get() with retry+backoff. Returns True on success, False if the
181
- page never loaded after all retries (caller should skip and move on)."""
182
- for attempt in range(max_retries):
183
- try:
184
- driver.get(url)
185
- return True
186
- except WebDriverException as e:
187
- delay = NETWORK_RETRY_BASE_DELAY * (2 ** attempt)
188
- print(f" [network error, retry {attempt+1}/{max_retries} in {delay}s] {url}")
189
- print(f" {str(e).splitlines()[0]}")
190
- time.sleep(delay)
191
- return False
192
-
193
- def crawl_and_download(driver, sess, seed_urls):
194
- visited_pages = set()
195
- pdf_links_found = set()
196
- downloaded_count = 0
197
- empty_seeds = []
198
- failed_seeds = []
199
- queue = list(seed_urls)
200
- login_warned = False
201
-
202
- pbar = tqdm(total=min(MAX_PAGES_TO_CRAWL, len(queue)), desc="Crawling seeded pages")
203
- while queue and len(visited_pages) < MAX_PAGES_TO_CRAWL:
204
- url = queue.pop(0)
205
- if url in visited_pages or not is_same_domain(url):
206
- continue
207
-
208
- visited_pages.add(url)
209
- pbar.update(1)
210
-
211
- page_t0 = time.time()
212
- if not safe_driver_get(driver, url):
213
- print(f" [gave up after {MAX_NETWORK_RETRIES} retries] {url}")
214
- failed_seeds.append(url)
215
- continue
216
- time.sleep(PAGE_LOAD_WAIT_SECONDS)
217
- page_elapsed = time.time() - page_t0
218
- if page_elapsed > 8:
219
- print(f" [slow page: {page_elapsed:.1f}s] {url}")
220
-
221
- html_lower = driver.page_source.lower()
222
-
223
- if looks_like_login_page(html_lower) and not login_warned:
224
- print(f"\n WARNING: {url} looks like a login page — your Edge session may not be logged in.")
225
- print(" Log in manually in the attached Edge window, then rerun.\n")
226
- login_warned = True
227
-
228
- if looks_like_error_or_empty(html_lower):
229
- empty_seeds.append(url)
230
-
231
- # Exhaust pagination before parsing the final DOM
232
- clicks = click_load_more_until_done(driver)
233
- if clicks:
234
- pbar.set_postfix(last_clicks=clicks)
235
-
236
- soup = BeautifulSoup(driver.page_source, "html.parser")
237
- page_pdf_count = 0
238
- for a in soup.find_all("a", href=True):
239
- href = urljoin(url, a["href"]).split("#")[0]
240
- href_path = urlparse(href).path.lower()
241
- if href_path.endswith(".pdf"):
242
- page_pdf_count += 1
243
- if href not in pdf_links_found:
244
- pdf_links_found.add(href)
245
- if download_one_pdf(sess, href):
246
- downloaded_count += 1
247
- pbar.set_postfix(pdfs=downloaded_count)
248
- elif is_same_domain(href) and href not in visited_pages:
249
- # Only follow same-domain links that are themselves other
250
- # Downloads.aspx pages (e.g. sub-folder drilldowns) — avoids
251
- # wandering off into unrelated intranet pages.
252
- if "downloads.aspx" in href.lower():
253
- queue.append(href)
254
-
255
- if page_pdf_count == 0:
256
- print(f" [0 pdfs] {url}")
257
-
258
- time.sleep(REQUEST_DELAY_SECONDS)
259
-
260
- pbar.close()
261
- print(f"\nCrawled {len(visited_pages)} pages. Found {len(pdf_links_found)} PDF links, downloaded {downloaded_count}.")
262
- if empty_seeds:
263
- print(f"\n{len(empty_seeds)} seed URLs looked empty/errored — check library= spelling for these:")
264
- for u in empty_seeds:
265
- print(f" {u}")
266
- if failed_seeds:
267
- print(f"\n{len(failed_seeds)} seed URLs failed after {MAX_NETWORK_RETRIES} retries (likely WiFi/DNS drop) — rerun these:")
268
- for u in failed_seeds:
269
- print(f" {u}")
270
- (OUTPUT_DIR / "failed_seeds.txt").write_text("\n".join(failed_seeds), encoding="utf-8")
271
- print(f" Written to {OUTPUT_DIR / 'failed_seeds.txt'}")
272
- return pdf_links_found
273
-
274
- # ---------------------------------------------------------------------------
275
- # Download PDFs — reuse the browser's cookies in a requests.Session so
276
- # binary downloads are fast and don't go through the browser's download UI
277
- # ---------------------------------------------------------------------------
278
- def session_from_driver(driver):
279
- sess = requests.Session()
280
- # Cookies alone won't authenticate against Windows Integrated Auth (NTLM/
281
- # Kerberos) sites — Edge handles that handshake transparently because
282
- # it's domain-joined, but a plain requests.Session never does the
283
- # handshake and gets 401s on every resource. Cloning cookies is kept as
284
- # a harmless supplement in case any part of the site is cookie-based.
285
- for cookie in driver.get_cookies():
286
- sess.cookies.set(cookie["name"], cookie["value"], domain=cookie.get("domain"))
287
- sess.headers.update({"User-Agent": driver.execute_script("return navigator.userAgent;")})
288
- if HAS_SSPI:
289
- sess.auth = HttpNegotiateAuth()
290
- else:
291
- print("WARNING: requests_negotiate_sspi not installed.")
292
- print(" Run: pip install requests-negotiate-sspi")
293
- print(" Without it, PDF downloads will likely 401 on this NTLM/Kerberos intranet.")
294
- return sess
295
-
296
- def safe_filename(url):
297
- name = os.path.basename(urlparse(url).path) or "document.pdf"
298
- name = re.sub(r"[^\w\-.]", "_", name)
299
- if not name.lower().endswith(".pdf"):
300
- name += ".pdf"
301
- url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
302
- return f"{url_hash}_{name}"
303
-
304
- def download_one_pdf(sess, url, max_retries=MAX_NETWORK_RETRIES):
305
- """Downloads a single PDF immediately. Returns True if a new file was saved."""
306
- PDF_DIR.mkdir(parents=True, exist_ok=True)
307
- dest = PDF_DIR / safe_filename(url)
308
- if dest.exists():
309
- return False
310
-
311
- for attempt in range(max_retries):
312
- try:
313
- resp = sess.get(url, timeout=REQUEST_TIMEOUT)
314
- resp.raise_for_status()
315
- if b"%PDF" not in resp.content[:1024]:
316
- print(f" [not a PDF, likely an auth redirect] {url}")
317
- return False
318
- dest.write_bytes(resp.content)
319
- return True
320
- except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
321
- delay = NETWORK_RETRY_BASE_DELAY * (2 ** attempt)
322
- print(f" [network error, retry {attempt+1}/{max_retries} in {delay}s] {url}")
323
- time.sleep(delay)
324
- except requests.RequestException as e:
325
- print(f" [failed] {url} ({e})")
326
- return False
327
- print(f" [gave up after {max_retries} retries] {url}")
328
- return False
329
-
330
- # ---------------------------------------------------------------------------
331
- # Extract text + tables from each PDF, with OCR fallback
332
- # ---------------------------------------------------------------------------
333
- def extract_pdf(pdf_path):
334
- """Returns (text, tables, used_ocr). Tries pdfplumber first; if a page's
335
- text comes back essentially empty (common for scanned/image-only PDFs),
336
- falls back to OCR for the whole document."""
337
- text_parts = []
338
- tables = []
339
- total_chars = 0
340
-
341
- try:
342
- with pdfplumber.open(pdf_path) as pdf:
343
- for page_num, page in enumerate(pdf.pages, 1):
344
- page_text = page.extract_text() or ""
345
- total_chars += len(page_text.strip())
346
- text_parts.append(f"\n--- Page {page_num} ---\n{page_text}")
347
-
348
- for table in page.extract_tables():
349
- if table:
350
- tables.append({"page": page_num, "rows": table})
351
- except Exception as e:
352
- print(f" [pdfplumber error] {pdf_path.name}: {e}")
353
-
354
- # Heuristic: if we got almost no text relative to file size, it's likely
355
- # a scanned/image PDF — fall back to OCR.
356
- avg_chars_per_page = total_chars / max(len(text_parts), 1)
357
- used_ocr = False
358
- if avg_chars_per_page < 20:
359
- try:
360
- ocr_text = _ocr_pdf(pdf_path)
361
- if len(ocr_text.strip()) > total_chars:
362
- text_parts = [ocr_text]
363
- used_ocr = True
364
- except Exception as e:
365
- print(f" [OCR unavailable/failed] {pdf_path.name}: {e}")
366
-
367
- return "\n".join(text_parts).strip(), tables, used_ocr
368
-
369
- def extract_all(downloaded):
370
- records = []
371
- for url, pdf_path in tqdm(downloaded, desc="Extracting text"):
372
- text, tables, used_ocr = extract_pdf(pdf_path)
373
- records.append({
374
- "source_url": url,
375
- "local_file": str(pdf_path),
376
- "text": text,
377
- "tables": tables,
378
- "used_ocr": used_ocr,
379
- "char_count": len(text),
380
- })
381
- return records
382
-
383
- # ---------------------------------------------------------------------------
384
- # Main
385
- # ---------------------------------------------------------------------------
386
- def main():
387
- OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
388
-
389
- print(f"Attaching to Edge on {EDGE_DEBUGGER_ADDRESS} ...")
390
- try:
391
- driver = get_driver()
392
- except Exception as e:
393
- print(f"Could not attach to Edge: {e}")
394
- print("Make sure Edge is running with --remote-debugging-port=9222 and you're logged into iamhbl.com.")
395
- return
396
-
397
- sess = session_from_driver(driver)
398
-
399
- print(f"Seeding {len(SEED_URLS)} function x library pages ({len(FUNCTIONS)} functions x {len(LIBRARIES)} libraries).")
400
- print("PDFs will be saved to the pdfs folder as they're found — check there anytime.\n")
401
- pdf_links = crawl_and_download(driver, sess, SEED_URLS)
402
-
403
- if not pdf_links:
404
- print("No PDFs found — check the login warning above, the empty-seed list, and confirm library= spelling.")
405
- return
406
-
407
- # Build the list of successfully downloaded files from disk (covers files
408
- # downloaded on this run and any left over from a previous interrupted run)
409
- downloaded = []
410
- for url in pdf_links:
411
- dest = PDF_DIR / safe_filename(url)
412
- if dest.exists():
413
- downloaded.append((url, dest))
414
- print(f"\n{len(downloaded)}/{len(pdf_links)} PDFs available on disk for extraction.\n")
415
-
416
- records = extract_all(downloaded)
417
-
418
- out_path = OUTPUT_DIR / "hbl_pdf_extracted.json"
419
- with open(out_path, "w", encoding="utf-8") as f:
420
- json.dump(records, f, ensure_ascii=False, indent=2)
421
-
422
- ocr_count = sum(1 for r in records if r["used_ocr"])
423
- empty_count = sum(1 for r in records if r["char_count"] < 20)
424
-
425
- print(f"\nDone. Extracted {len(records)} PDFs.")
426
- print(f" - {ocr_count} needed OCR fallback (scanned/image PDFs)")
427
- print(f" - {empty_count} still came back nearly empty — worth manually checking these")
428
- print(f" - Metadata written to {out_path}")
429
-
430
- if __name__ == "__main__":
431
- main()
 
1
+ (venv) PS D:\AI\chatbot> & "C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe" --remote-debugging-port=9222
2
+ (venv) PS D:\AI\chatbot> python crawler.py
3
+ Attaching to Edge on 127.0.0.1:9222 ...
4
+ Could not attach to Edge: Message: session not created: cannot connect to microsoft edge at 127.0.0.1:9222
5
+ from chrome not reachable; For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#sessionnotcreatedexception
6
+ Stacktrace:
7
+ msedgedriver!GetHandleVerifier [0x7ff6976874c5+e415]
8
+ msedgedriver!GetHandleVerifier [0x7ff697687524+e474]
9
+ msedgedriver!(No symbol) [0x7ff69703614e]
10
+ msedgedriver!(No symbol) [0x7ff69702bd0a]
11
+ msedgedriver!(No symbol) [0x7ff697072a74]
12
+ msedgedriver!(No symbol) [0x7ff697068f37]
13
+ msedgedriver!(No symbol) [0x7ff6970abaf3]
14
+ msedgedriver!(No symbol) [0x7ff6970ab486]
15
+ msedgedriver!(No symbol) [0x7ff697076bfc]
16
+ msedgedriver!(No symbol) [0x7ff697075e56]
17
+ msedgedriver!(No symbol) [0x7ff697076a23]
18
+ msedgedriver!(No symbol) [0x7ff6972b1131]
19
+ msedgedriver!(No symbol) [0x7ff6972ad54f]
20
+ msedgedriver!(No symbol) [0x7ff6972be3d9]
21
+ msedgedriver!GetHandleVerifier [0x7ff6976a2bb1+29b01]
22
+ msedgedriver!GetHandleVerifier [0x7ff6976ab3b6+32306]
23
+ msedgedriver!GetHandleVerifier [0x7ff69768f024+15f74]
24
+ msedgedriver!GetHandleVerifier [0x7ff69768f145+16095]
25
+ msedgedriver!GetHandleVerifier [0x7ff69767b563+24b3]
26
+ KERNEL32!BaseThreadInitThunk [0x7ffa6e5de8d7+17]
27
+ ntdll!RtlUserThreadStart [0x7ffa6f5ac53c+2c]
28
+
29
+ Make sure Edge is running with --remote-debugging-port=9222 and you're logged into iamhbl.com.
30
+ (venv) PS D:\AI\chatbot>