Download dev/crawling.py from m8chaa/moyo-crawling-selenium-test: direct link, hf CLI and curl.
- Browser
- Download file 6.31 kB
-
https://huggingface.co/spaces/m8chaa/moyo-crawling-selenium-test/resolve/main/dev/crawling.py
- Command line
-
hf download hf://spaces/m8chaa/moyo-crawling-selenium-test/dev/crawling.py
-
curl -L -o crawling.py https://huggingface.co/spaces/m8chaa/moyo-crawling-selenium-test/resolve/main/dev/crawling.py
6.31 kB
| import threading | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import logging | |
| from queue import Queue | |
| import time | |
| import psutil | |
| from .processing import process_data, retry_push_to_sheet | |
| from .selenium_utils import setup_driver, interact_with_page_selenium | |
| from .google_sheet_utils import auto_resize_columns | |
| def fetch_url_just_moyos(url_fetch_queue, page_counter, end_page, counter_lock, active_fetch_threads, num_threads): | |
| base_url = "https://www.moyoplan.com" | |
| end_of_list = False | |
| while not end_of_list: | |
| with counter_lock: | |
| if page_counter[0] > end_page: | |
| break | |
| i = page_counter[0] | |
| page_counter[0] += 1 | |
| attempts = 0 | |
| while attempts < 5: | |
| try: | |
| plan_list_url = f"{base_url}/plans?sorting=fee_asc&page={i}" | |
| response = requests.get(plan_list_url) | |
| if response.status_code != 200: | |
| logging.error(f"Failed to fetch data from {plan_list_url}. Status code: {response.status_code}") | |
| attempts += 1 | |
| if attempts >= 5: | |
| logging.error(f"Max attempts reached for {plan_list_url}. Skipping...") | |
| break | |
| soup = BeautifulSoup(response.text, 'lxml') | |
| div_tags = soup.find_all('div', class_='css-aly04n e12j53ye0') | |
| for div in div_tags: | |
| a_tag = div.find('a') | |
| if not a_tag: | |
| continue | |
| link = a_tag['href'] | |
| plan_detail_url = f"{base_url}{link}" | |
| url_fetch_queue.put(plan_detail_url) | |
| if not div_tags: | |
| end_of_list = True | |
| break # End the while True loop if no more div tags are found | |
| # No break here, continue to the next page | |
| break | |
| except Exception as e: | |
| logging.error(str(e)) | |
| attempts += 1 | |
| if attempts >= 5: | |
| break | |
| logging.info(f"URL Fetch Thread Finished at page = {i}") | |
| with counter_lock: | |
| active_fetch_threads[0] -= 1 | |
| if active_fetch_threads[0] == 0: | |
| for _ in range(num_threads*3): | |
| url_fetch_queue.put(None) # Signal the end of the queue | |
| def fetch_data_just_moyos(url_fetch_queue, data_queue, error_queue): | |
| while True: | |
| url = url_fetch_queue.get() | |
| if url is None: | |
| url_fetch_queue.put(None) # Signal the next thread | |
| url_fetch_queue.task_done() | |
| break | |
| try: | |
| response = requests.get(url) | |
| soup = BeautifulSoup(response.text, 'lxml') | |
| script_tag = soup.find('script', {'id': '__NEXT_DATA__'}) | |
| if script_tag: | |
| data = script_tag.contents[0] | |
| data_queue.put(data) | |
| url_fetch_queue.task_done() | |
| except Exception as e: | |
| error_queue.put(f"Error in fetch_data_just_moyos: {str(e)}") | |
| logging.error(f"Exception in URL fetch process: {e}") | |
| url_fetch_queue.task_done() | |
| url_fetch_queue.put(url) | |
| def update_sheet(processed_data_queue, sheet_update_lock, sheet_id, service_instance, error_queue): | |
| while True: | |
| batch_data = [] | |
| while len(batch_data) < 100: | |
| processed_data = processed_data_queue.get() | |
| if processed_data is None: | |
| if batch_data: | |
| with sheet_update_lock: | |
| retry_push_to_sheet(batch_data, sheet_id, 'Sheet3!A2', service_instance) | |
| with sheet_update_lock: | |
| retry_push_to_sheet([["Fetch data ended successfully"]], sheet_id, 'CrawlingControl!B2', service_instance) | |
| return | |
| batch_data.append(processed_data) | |
| with sheet_update_lock: | |
| retry_push_to_sheet(batch_data, sheet_id, 'Sheet3!A2', service_instance) | |
| def moyocrawling_just_moyos(sheet_id, sheet_url, service_instance, error_queue, log_queue, thread_completed, num_threads, initial_backup_thread): | |
| url_fetch_queue = Queue() | |
| data_queue = Queue() | |
| processed_data_queue = Queue() | |
| sheet_update_lock = threading.Lock() | |
| counter_lock = threading.Lock() | |
| page_counter = [1] # Shared counter | |
| active_fetch_threads = [num_threads] # Counter for active fetch threads | |
| end_page = float('inf') | |
| # Create URL fetch threads based on num_threads | |
| url_fetch_threads = [threading.Thread(target=fetch_url_just_moyos, args=(url_fetch_queue, page_counter, end_page, counter_lock, active_fetch_threads, num_threads)) for _ in range(num_threads)] | |
| data_fetch_threads = [threading.Thread(target=fetch_data_just_moyos, args=(url_fetch_queue, data_queue, error_queue)) for _ in range(num_threads*3)] | |
| process_data_threads = [threading.Thread(target=process_data, args=(data_queue, processed_data_queue, error_queue)) for _ in range(num_threads*3)] | |
| update_sheet_thread = threading.Thread(target=update_sheet, args=(processed_data_queue, sheet_update_lock, sheet_id, service_instance, error_queue)) | |
| for thread in url_fetch_threads: | |
| thread.start() | |
| logging.info("Fetch URL Threads Started") | |
| for thread in data_fetch_threads: | |
| thread.start() | |
| logging.info("Fetch Data Thread Started") | |
| for thread in process_data_threads: | |
| thread.start() | |
| logging.info("Process Data Thread Started") | |
| initial_backup_thread.join() | |
| update_sheet_thread.start() | |
| logging.info("Update Sheet Thread Started") | |
| for thread in url_fetch_threads: | |
| thread.join() | |
| logging.info("Fetch URL Threads Completed") | |
| # Ensure we only put None once per data fetch thread | |
| for _ in data_fetch_threads: | |
| url_fetch_queue.put(None) | |
| for thread in data_fetch_threads: | |
| thread.join() | |
| logging.info("Fetch Data Thread Completed") | |
| for _ in process_data_threads: | |
| data_queue.put(None) | |
| for thread in process_data_threads: | |
| thread.join() | |
| logging.info("Process Data Thread Completed") | |
| update_sheet_thread.join() | |
| logging.info("Update Sheet Thread Completed") | |
| thread_completed.set() | |
| logging.info("All Threads Completed") | |