import threading import requests from bs4 import BeautifulSoup import logging from queue import Queue import time import psutil from .processing import process_data, retry_push_to_sheet from .selenium_utils import setup_driver, interact_with_page_selenium from .google_sheet_utils import auto_resize_columns def fetch_url_just_moyos(url_fetch_queue, page_counter, end_page, counter_lock, active_fetch_threads, num_threads): base_url = "https://www.moyoplan.com" end_of_list = False while not end_of_list: with counter_lock: if page_counter[0] > end_page: break i = page_counter[0] page_counter[0] += 1 attempts = 0 while attempts < 5: try: plan_list_url = f"{base_url}/plans?sorting=fee_asc&page={i}" response = requests.get(plan_list_url) if response.status_code != 200: logging.error(f"Failed to fetch data from {plan_list_url}. Status code: {response.status_code}") attempts += 1 if attempts >= 5: logging.error(f"Max attempts reached for {plan_list_url}. Skipping...") break soup = BeautifulSoup(response.text, 'lxml') div_tags = soup.find_all('div', class_='css-aly04n e12j53ye0') for div in div_tags: a_tag = div.find('a') if not a_tag: continue link = a_tag['href'] plan_detail_url = f"{base_url}{link}" url_fetch_queue.put(plan_detail_url) if not div_tags: end_of_list = True break # End the while True loop if no more div tags are found # No break here, continue to the next page break except Exception as e: logging.error(str(e)) attempts += 1 if attempts >= 5: break logging.info(f"URL Fetch Thread Finished at page = {i}") with counter_lock: active_fetch_threads[0] -= 1 if active_fetch_threads[0] == 0: for _ in range(num_threads*3): url_fetch_queue.put(None) # Signal the end of the queue def fetch_data_just_moyos(url_fetch_queue, data_queue, error_queue): while True: url = url_fetch_queue.get() if url is None: url_fetch_queue.put(None) # Signal the next thread url_fetch_queue.task_done() break try: response = requests.get(url) soup = BeautifulSoup(response.text, 'lxml') script_tag = soup.find('script', {'id': '__NEXT_DATA__'}) if script_tag: data = script_tag.contents[0] data_queue.put(data) url_fetch_queue.task_done() except Exception as e: error_queue.put(f"Error in fetch_data_just_moyos: {str(e)}") logging.error(f"Exception in URL fetch process: {e}") url_fetch_queue.task_done() url_fetch_queue.put(url) def update_sheet(processed_data_queue, sheet_update_lock, sheet_id, service_instance, error_queue): while True: batch_data = [] while len(batch_data) < 100: processed_data = processed_data_queue.get() if processed_data is None: if batch_data: with sheet_update_lock: retry_push_to_sheet(batch_data, sheet_id, 'Sheet3!A2', service_instance) with sheet_update_lock: retry_push_to_sheet([["Fetch data ended successfully"]], sheet_id, 'CrawlingControl!B2', service_instance) return batch_data.append(processed_data) with sheet_update_lock: retry_push_to_sheet(batch_data, sheet_id, 'Sheet3!A2', service_instance) def moyocrawling_just_moyos(sheet_id, sheet_url, service_instance, error_queue, log_queue, thread_completed, num_threads, initial_backup_thread): url_fetch_queue = Queue() data_queue = Queue() processed_data_queue = Queue() sheet_update_lock = threading.Lock() counter_lock = threading.Lock() page_counter = [1] # Shared counter active_fetch_threads = [num_threads] # Counter for active fetch threads end_page = float('inf') # Create URL fetch threads based on num_threads url_fetch_threads = [threading.Thread(target=fetch_url_just_moyos, args=(url_fetch_queue, page_counter, end_page, counter_lock, active_fetch_threads, num_threads)) for _ in range(num_threads)] data_fetch_threads = [threading.Thread(target=fetch_data_just_moyos, args=(url_fetch_queue, data_queue, error_queue)) for _ in range(num_threads*3)] process_data_threads = [threading.Thread(target=process_data, args=(data_queue, processed_data_queue, error_queue)) for _ in range(num_threads*3)] update_sheet_thread = threading.Thread(target=update_sheet, args=(processed_data_queue, sheet_update_lock, sheet_id, service_instance, error_queue)) for thread in url_fetch_threads: thread.start() logging.info("Fetch URL Threads Started") for thread in data_fetch_threads: thread.start() logging.info("Fetch Data Thread Started") for thread in process_data_threads: thread.start() logging.info("Process Data Thread Started") initial_backup_thread.join() update_sheet_thread.start() logging.info("Update Sheet Thread Started") for thread in url_fetch_threads: thread.join() logging.info("Fetch URL Threads Completed") # Ensure we only put None once per data fetch thread for _ in data_fetch_threads: url_fetch_queue.put(None) for thread in data_fetch_threads: thread.join() logging.info("Fetch Data Thread Completed") for _ in process_data_threads: data_queue.put(None) for thread in process_data_threads: thread.join() logging.info("Process Data Thread Completed") update_sheet_thread.join() logging.info("Update Sheet Thread Completed") thread_completed.set() logging.info("All Threads Completed")