Download Crawler.py from gkim93/PID: direct link, hf CLI and curl.
- Browser
- Download file 3.96 kB
-
https://huggingface.co/spaces/gkim93/PID/resolve/main/Crawler.py
- Command line
-
hf download hf://spaces/gkim93/PID/Crawler.py
-
curl -L -o Crawler.py https://huggingface.co/spaces/gkim93/PID/resolve/main/Crawler.py
3.96 kB
| # -*- coding: utf-8 -*- | |
| """ | |
| Created on Tue Jul 25 11:41:43 2023 | |
| This Python Script to Crawl PDF links from ADB | |
| @author: intern.giwon.kim | |
| """ | |
| from urllib import request | |
| from bs4 import BeautifulSoup | |
| import re | |
| import os | |
| import urllib | |
| import random | |
| # ------------------Proxy setting-------------------# | |
| # Default value is 0. You can set to 1 if you have proxy and fill the tunnel, username and password below. | |
| usingProxy = 0 | |
| tunnel = 'tunnel.qg.net:45129' | |
| username = "992272B6" | |
| password = "E6E199CC6B76" | |
| # Define agent and opener to open a web | |
| def header_define(usingProxy): | |
| """ | |
| Multiple headers could be added in case of retrieval failure. | |
| :return: an opener for machine to open a web. | |
| It is an inner build type. | |
| """ | |
| my_headers = [ | |
| "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36", | |
| "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", | |
| "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36", | |
| "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14", | |
| "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)", | |
| 'Opera/9.25 (Windows NT 5.1; U; en)', | |
| 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', | |
| 'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)', | |
| 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9', | |
| "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7"] | |
| if usingProxy: | |
| proxy_support = urllib.request.ProxyHandler({'https': 'http://%(user)s:%(pwd)s@%(proxy)s' % {"user": username, | |
| "pwd": password, | |
| "proxy": tunnel}}) | |
| opener = urllib.request.build_opener(proxy_support) | |
| opener.addheaders = [{'User-Agent', random.choice(my_headers)}] | |
| else: | |
| opener = urllib.request.build_opener() | |
| opener.addheaders = [{'User-Agent', random.choice(my_headers)}] | |
| return opener | |
| def launch_webdriver(web_driver = "Edge"): | |
| # Choose your browse. For AIIB users, the microsoft Edge is recommended. | |
| if web_driver == "Edge": | |
| try: | |
| driver = webdriver.Edge(r"../lib/msedgedriver.exe") | |
| except Exception as e: | |
| print(e) | |
| if "version" in e: | |
| print("Please download the latest version from https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/ . Choose The 'stable channle', 'X64' version.") | |
| print("If you have download and save it to './lib' folder, input 'y', if you want to quit, input 'q':") | |
| if input().strip() == "y": | |
| launch_webdriver() | |
| if input().strip() == "q": | |
| sys.exit() | |
| else: | |
| print("Please make sure there is a ./lib folder under current space. And check if there is a msedgedriver.exe file.\ If not, please download from: https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/ . Choose The 'stable channle', 'X64' version.") | |
| return driver | |
| header = header_define(0) | |
| driver = launch_webdriver() | |
| # connect to website and get list of all pdfs | |
| soup= BeautifulSoup(driver.page_source, "html.parser") | |
| links = soup.find_all('a', href=re.compile(r'(.pdf)')) | |
| # clean the pdf link names | |
| url_list = [] | |
| for el in links: | |
| url_list.append(("https://www.adb.org/sites/default/files/project-documents/" + el['href'])) | |
| print(url_list) | |