moyo-crawling-selenium-test / dev /selenium_utils.py
m8chaa's picture
multiprocessing
c1edaec
Raw History Blame Contribute Delete
3.67 kB
from selenium import webdriver
from selenium.webdriver.chrome import service as fs
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException
from webdriver_manager.chrome import ChromeDriverManager, ChromeType
import logging
import threading
import speedtest
import time
CHROMEDRIVER = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()
def setup_driver():
options = ChromeOptions()
options.add_argument("--headless")
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-extensions')
prefs = {"profile.managed_default_content_settings.images": 2}
options.add_experimental_option("prefs", prefs)
service = fs.Service(CHROMEDRIVER)
driver = webdriver.Chrome(
options=options,
service=service
)
return driver
def retry_interaction(driver, url, data_queue, url_fetch_queue, max_retries, retries):
try:
logging.info(f"Retrying interaction with page: {url} (Attempt {retries + 1})")
interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries, retries)
except Exception as e:
logging.error(f"Unexpected error during retry of {url}: {str(e)}")
def interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries=3, retries=0):
try:
logging.info(f"Waiting for presence of element with class 'css-yg1ktq' on page {url}")
WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CLASS_NAME, "css-yg1ktq")))
logging.info(f"Waiting for element with class 'css-yg1ktq' to be clickable on page {url}")
WebDriverWait(driver, (retries ** 2) + 2).until(EC.element_to_be_clickable((By.CLASS_NAME, "css-yg1ktq")))
logging.info(f"Finding button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}")
button = driver.find_element(By.XPATH, "//button[contains(@class, 'css-yg1ktq')]")
logging.info(f"Clicking button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}")
driver.execute_script("arguments[0].click();", button)
logging.info(f"Waiting for presence of element with CSS selector 'div' on page {url}")
WebDriverWait(driver, (retries ** 2) + 2).until(lambda d: d.execute_script('return document.readyState') == 'complete')
WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div')))
logging.info(f"Getting page source for page {url}")
html = driver.page_source
data_queue.put((url, html))
except (TimeoutException, NoSuchElementException, StaleElementReferenceException) as e:
logging.error(f"Error interacting with page {url}: {str(e)} - Retry {retries + 1}/{max_retries}")
if retries < max_retries:
retries += 1
retry_thread = threading.Thread(target=retry_interaction, args=(driver, url, data_queue, url_fetch_queue, max_retries, retries))
retry_thread.start()
else:
logging.error(f"Max retries reached for {url}. Re-queueing...")
put_url_first_in_queue(url_fetch_queue, url)
except Exception as e:
logging.error(f"Unexpected error interacting with page {url}: {str(e)}")
def put_url_first_in_queue(queue, url):
queue.put(url)