File size: 3,665 Bytes
557d2ff
 
 
 
 
 
e40c6d8
557d2ff
 
 
8ffcf89
fe9b040
557d2ff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d3d9c99
7c7688a
d3d9c99
 
557d2ff
d3d9c99
 
557d2ff
d3d9c99
 
557d2ff
d3d9c99
 
c1edaec
557d2ff
d3d9c99
 
557d2ff
d3d9c99
557d2ff
e40c6d8
557d2ff
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
from selenium import webdriver
from selenium.webdriver.chrome import service as fs
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException
from webdriver_manager.chrome import ChromeDriverManager, ChromeType
import logging
import threading
import speedtest
import time

CHROMEDRIVER = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()

def setup_driver():
    options = ChromeOptions()
    options.add_argument("--headless")
    options.add_argument('--disable-gpu')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('--disable-extensions')
    prefs = {"profile.managed_default_content_settings.images": 2}
    options.add_experimental_option("prefs", prefs)
    service = fs.Service(CHROMEDRIVER)
    driver = webdriver.Chrome(
        options=options,
        service=service
    )
    return driver

def retry_interaction(driver, url, data_queue, url_fetch_queue, max_retries, retries):
    try:
        logging.info(f"Retrying interaction with page: {url} (Attempt {retries + 1})")
        interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries, retries)
    except Exception as e:
        logging.error(f"Unexpected error during retry of {url}: {str(e)}")

def interact_with_page_selenium(driver, url, data_queue, url_fetch_queue, max_retries=3, retries=0):
    try:
        logging.info(f"Waiting for presence of element with class 'css-yg1ktq' on page {url}")
        WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CLASS_NAME, "css-yg1ktq")))

        logging.info(f"Waiting for element with class 'css-yg1ktq' to be clickable on page {url}")
        WebDriverWait(driver, (retries ** 2) + 2).until(EC.element_to_be_clickable((By.CLASS_NAME, "css-yg1ktq")))

        logging.info(f"Finding button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}")
        button = driver.find_element(By.XPATH, "//button[contains(@class, 'css-yg1ktq')]")

        logging.info(f"Clicking button with xpath '//button[contains(@class, 'css-yg1ktq')]' on page {url}")
        driver.execute_script("arguments[0].click();", button)

        logging.info(f"Waiting for presence of element with CSS selector 'div' on page {url}")
        WebDriverWait(driver, (retries ** 2) + 2).until(lambda d: d.execute_script('return document.readyState') == 'complete')
        WebDriverWait(driver, (retries ** 2) + 2).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div')))

        logging.info(f"Getting page source for page {url}")
        html = driver.page_source

        data_queue.put((url, html))
    except (TimeoutException, NoSuchElementException, StaleElementReferenceException) as e:
        logging.error(f"Error interacting with page {url}: {str(e)} - Retry {retries + 1}/{max_retries}")
        if retries < max_retries:
            retries += 1
            retry_thread = threading.Thread(target=retry_interaction, args=(driver, url, data_queue, url_fetch_queue, max_retries, retries))
            retry_thread.start()
        else:
            logging.error(f"Max retries reached for {url}. Re-queueing...")
            put_url_first_in_queue(url_fetch_queue, url)
    except Exception as e:
        logging.error(f"Unexpected error interacting with page {url}: {str(e)}")

def put_url_first_in_queue(queue, url):
    queue.put(url)