Download dev/selenium_utils.py from m8chaa/moyo-crawling-requests: direct link, hf CLI and curl.
- Browser
- Download file 3.67 kB
-
https://huggingface.co/spaces/m8chaa/moyo-crawling-requests/resolve/main/dev/selenium_utils.py
- Command line
-
hf download hf://spaces/m8chaa/moyo-crawling-requests/dev/selenium_utils.py
-
curl -L -o selenium_utils.py https://huggingface.co/spaces/m8chaa/moyo-crawling-requests/resolve/main/dev/selenium_utils.py
3.67 kB
| from selenium import webdriver | |
| from selenium.webdriver.chrome import service as fs | |
| from selenium.webdriver import ChromeOptions | |
| from selenium.webdriver.common.by import By | |
| from selenium.webdriver.support.ui import WebDriverWait | |
| from selenium.webdriver.support import expected_conditions as EC | |
| from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException | |
| from webdriver_manager.chrome import ChromeDriverManager, ChromeType | |
| import logging | |
| import threading | |
| import speedtest | |
| import time | |
| CHROMEDRIVER = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install() | |
| def setup_driver(): | |
| options = ChromeOptions() | |
| options.add_argument("--headless") | |
| options.add_argument('--disable-gpu') | |
| options.add_argument('--no-sandbox') | |
| options.add_argument('--disable-dev-shm-usage') | |
| options.add_argument('--disable-extensions') | |
| prefs = {"profile.managed_default_content_settings.images": 2} | |
| options.add_experimental_option("prefs", prefs) | |
| service = fs.Service(CHROMEDRIVER) | |
| driver = webdriver.Chrome( | |
| options=options, | |
| service=service | |
| ) | |
| return driver | |
| def retry_interaction(driver, url, data_queue, url_fetch_queue, max_retries, retries): | |
| try: | |
| logging.info(f"Retrying interaction with page: {url} (Attempt {retries + 1})") | |
| interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries, retries) | |
| except Exception as e: | |
| logging.error(f"Unexpected error during retry of {url}: {str(e)}") | |
| def interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries=3, retries=0): | |
| try: | |
| logging.info(f"Waiting for presence of element with class 'css-yg1ktq' on page {url}") | |
| WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CLASS_NAME, "css-yg1ktq"))) | |
| logging.info(f"Waiting for element with class 'css-yg1ktq' to be clickable on page {url}") | |
| WebDriverWait(driver, (retries ** 2) + 2).until(EC.element_to_be_clickable((By.CLASS_NAME, "css-yg1ktq"))) | |
| logging.info(f"Finding button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}") | |
| button = driver.find_element(By.XPATH, "//button[contains(@class, 'css-yg1ktq')]") | |
| logging.info(f"Clicking button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}") | |
| driver.execute_script("arguments[0].click();", button) | |
| logging.info(f"Waiting for presence of element with CSS selector 'div' on page {url}") | |
| WebDriverWait(driver, (retries ** 2) + 2).until(lambda d: d.execute_script('return document.readyState') == 'complete') | |
| WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div'))) | |
| logging.info(f"Getting page source for page {url}") | |
| html = driver.page_source | |
| data_queue.put((url, html)) | |
| except (TimeoutException, NoSuchElementException, StaleElementReferenceException) as e: | |
| logging.error(f"Error interacting with page {url}: {str(e)} - Retry {retries + 1}/{max_retries}") | |
| if retries < max_retries: | |
| retries += 1 | |
| retry_thread = threading.Thread(target=retry_interaction, args=(driver, url, data_queue, url_fetch_queue, max_retries, retries)) | |
| retry_thread.start() | |
| else: | |
| logging.error(f"Max retries reached for {url}. Re-queueing...") | |
| put_url_first_in_queue(url_fetch_queue, url) | |
| except Exception as e: | |
| logging.error(f"Unexpected error interacting with page {url}: {str(e)}") | |
| def put_url_first_in_queue(queue, url): | |
| queue.put(url) | |