from selenium import webdriver from selenium.webdriver.chrome import service as fs from selenium.webdriver import ChromeOptions from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException from webdriver_manager.chrome import ChromeDriverManager, ChromeType import logging import threading import speedtest import time CHROMEDRIVER = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install() def setup_driver(): options = ChromeOptions() options.add_argument("--headless") options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--disable-extensions') prefs = {"profile.managed_default_content_settings.images": 2} options.add_experimental_option("prefs", prefs) service = fs.Service(CHROMEDRIVER) driver = webdriver.Chrome( options=options, service=service ) return driver def retry_interaction(driver, url, data_queue, url_fetch_queue, max_retries, retries): try: logging.info(f"Retrying interaction with page: {url} (Attempt {retries + 1})") interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries, retries) except Exception as e: logging.error(f"Unexpected error during retry of {url}: {str(e)}") def interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries=3, retries=0): try: logging.info(f"Waiting for presence of element with class 'css-yg1ktq' on page {url}") WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CLASS_NAME, "css-yg1ktq"))) logging.info(f"Waiting for element with class 'css-yg1ktq' to be clickable on page {url}") WebDriverWait(driver, (retries ** 2) + 2).until(EC.element_to_be_clickable((By.CLASS_NAME, "css-yg1ktq"))) logging.info(f"Finding button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}") button = driver.find_element(By.XPATH, "//button[contains(@class, 'css-yg1ktq')]") logging.info(f"Clicking button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}") driver.execute_script("arguments[0].click();", button) logging.info(f"Waiting for presence of element with CSS selector 'div' on page {url}") WebDriverWait(driver, (retries ** 2) + 2).until(lambda d: d.execute_script('return document.readyState') == 'complete') WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div'))) logging.info(f"Getting page source for page {url}") html = driver.page_source data_queue.put((url, html)) except (TimeoutException, NoSuchElementException, StaleElementReferenceException) as e: logging.error(f"Error interacting with page {url}: {str(e)} - Retry {retries + 1}/{max_retries}") if retries < max_retries: retries += 1 retry_thread = threading.Thread(target=retry_interaction, args=(driver, url, data_queue, url_fetch_queue, max_retries, retries)) retry_thread.start() else: logging.error(f"Max retries reached for {url}. Re-queueing...") put_url_first_in_queue(url_fetch_queue, url) except Exception as e: logging.error(f"Unexpected error interacting with page {url}: {str(e)}") def put_url_first_in_queue(queue, url): queue.put(url)