File size: 3,665 Bytes
557d2ff e40c6d8 557d2ff 8ffcf89 fe9b040 557d2ff d3d9c99 7c7688a d3d9c99 557d2ff d3d9c99 557d2ff d3d9c99 557d2ff d3d9c99 c1edaec 557d2ff d3d9c99 557d2ff d3d9c99 557d2ff e40c6d8 557d2ff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | from selenium import webdriver
from selenium.webdriver.chrome import service as fs
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException
from webdriver_manager.chrome import ChromeDriverManager, ChromeType
import logging
import threading
import speedtest
import time
CHROMEDRIVER = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()
def setup_driver():
options = ChromeOptions()
options.add_argument("--headless")
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-extensions')
prefs = {"profile.managed_default_content_settings.images": 2}
options.add_experimental_option("prefs", prefs)
service = fs.Service(CHROMEDRIVER)
driver = webdriver.Chrome(
options=options,
service=service
)
return driver
def retry_interaction(driver, url, data_queue, url_fetch_queue, max_retries, retries):
try:
logging.info(f"Retrying interaction with page: {url} (Attempt {retries + 1})")
interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries, retries)
except Exception as e:
logging.error(f"Unexpected error during retry of {url}: {str(e)}")
def interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries=3, retries=0):
try:
logging.info(f"Waiting for presence of element with class 'css-yg1ktq' on page {url}")
WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CLASS_NAME, "css-yg1ktq")))
logging.info(f"Waiting for element with class 'css-yg1ktq' to be clickable on page {url}")
WebDriverWait(driver, (retries ** 2) + 2).until(EC.element_to_be_clickable((By.CLASS_NAME, "css-yg1ktq")))
logging.info(f"Finding button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}")
button = driver.find_element(By.XPATH, "//button[contains(@class, 'css-yg1ktq')]")
logging.info(f"Clicking button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}")
driver.execute_script("arguments[0].click();", button)
logging.info(f"Waiting for presence of element with CSS selector 'div' on page {url}")
WebDriverWait(driver, (retries ** 2) + 2).until(lambda d: d.execute_script('return document.readyState') == 'complete')
WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div')))
logging.info(f"Getting page source for page {url}")
html = driver.page_source
data_queue.put((url, html))
except (TimeoutException, NoSuchElementException, StaleElementReferenceException) as e:
logging.error(f"Error interacting with page {url}: {str(e)} - Retry {retries + 1}/{max_retries}")
if retries < max_retries:
retries += 1
retry_thread = threading.Thread(target=retry_interaction, args=(driver, url, data_queue, url_fetch_queue, max_retries, retries))
retry_thread.start()
else:
logging.error(f"Max retries reached for {url}. Re-queueing...")
put_url_first_in_queue(url_fetch_queue, url)
except Exception as e:
logging.error(f"Unexpected error interacting with page {url}: {str(e)}")
def put_url_first_in_queue(queue, url):
queue.put(url)
|