m8chaa's picture
fetch url threads + num_threads:7
f07363b
Raw History Blame Contribute Delete
6.31 kB
import threading
import requests
from bs4 import BeautifulSoup
import logging
from queue import Queue
import time
import psutil
from .processing import process_data, retry_push_to_sheet
from .selenium_utils import setup_driver, interact_with_page_selenium
from .google_sheet_utils import auto_resize_columns
def fetch_url_just_moyos(url_fetch_queue, page_counter, end_page, counter_lock, active_fetch_threads, num_threads):
base_url = "https://www.moyoplan.com"
end_of_list = False
while not end_of_list:
with counter_lock:
if page_counter[0] > end_page:
break
i = page_counter[0]
page_counter[0] += 1
attempts = 0
while attempts < 5:
try:
plan_list_url = f"{base_url}/plans?sorting=fee_asc&page={i}"
response = requests.get(plan_list_url)
if response.status_code != 200:
logging.error(f"Failed to fetch data from {plan_list_url}. Status code: {response.status_code}")
attempts += 1
if attempts >= 5:
logging.error(f"Max attempts reached for {plan_list_url}. Skipping...")
break
soup = BeautifulSoup(response.text, 'lxml')
div_tags = soup.find_all('div', class_='css-aly04n e12j53ye0')
for div in div_tags:
a_tag = div.find('a')
if not a_tag:
continue
link = a_tag['href']
plan_detail_url = f"{base_url}{link}"
url_fetch_queue.put(plan_detail_url)
if not div_tags:
end_of_list = True
break # End the while True loop if no more div tags are found
# No break here, continue to the next page
break
except Exception as e:
logging.error(str(e))
attempts += 1
if attempts >= 5:
break
logging.info(f"URL Fetch Thread Finished at page = {i}")
with counter_lock:
active_fetch_threads[0] -= 1
if active_fetch_threads[0] == 0:
for _ in range(num_threads*3):
url_fetch_queue.put(None) # Signal the end of the queue
def fetch_data_just_moyos(url_fetch_queue, data_queue, error_queue):
while True:
url = url_fetch_queue.get()
if url is None:
url_fetch_queue.put(None) # Signal the next thread
url_fetch_queue.task_done()
break
try:
response = requests.get(url)
soup = BeautifulSoup(response.text, 'lxml')
script_tag = soup.find('script', {'id': '__NEXT_DATA__'})
if script_tag:
data = script_tag.contents[0]
data_queue.put(data)
url_fetch_queue.task_done()
except Exception as e:
error_queue.put(f"Error in fetch_data_just_moyos: {str(e)}")
logging.error(f"Exception in URL fetch process: {e}")
url_fetch_queue.task_done()
url_fetch_queue.put(url)
def update_sheet(processed_data_queue, sheet_update_lock, sheet_id, service_instance, error_queue):
while True:
batch_data = []
while len(batch_data) < 100:
processed_data = processed_data_queue.get()
if processed_data is None:
if batch_data:
with sheet_update_lock:
retry_push_to_sheet(batch_data, sheet_id, 'Sheet3!A2', service_instance)
with sheet_update_lock:
retry_push_to_sheet([["Fetch data ended successfully"]], sheet_id, 'CrawlingControl!B2', service_instance)
return
batch_data.append(processed_data)
with sheet_update_lock:
retry_push_to_sheet(batch_data, sheet_id, 'Sheet3!A2', service_instance)
def moyocrawling_just_moyos(sheet_id, sheet_url, service_instance, error_queue, log_queue, thread_completed, num_threads, initial_backup_thread):
url_fetch_queue = Queue()
data_queue = Queue()
processed_data_queue = Queue()
sheet_update_lock = threading.Lock()
counter_lock = threading.Lock()
page_counter = [1] # Shared counter
active_fetch_threads = [num_threads] # Counter for active fetch threads
end_page = float('inf')
# Create URL fetch threads based on num_threads
url_fetch_threads = [threading.Thread(target=fetch_url_just_moyos, args=(url_fetch_queue, page_counter, end_page, counter_lock, active_fetch_threads, num_threads)) for _ in range(num_threads)]
data_fetch_threads = [threading.Thread(target=fetch_data_just_moyos, args=(url_fetch_queue, data_queue, error_queue)) for _ in range(num_threads*3)]
process_data_threads = [threading.Thread(target=process_data, args=(data_queue, processed_data_queue, error_queue)) for _ in range(num_threads*3)]
update_sheet_thread = threading.Thread(target=update_sheet, args=(processed_data_queue, sheet_update_lock, sheet_id, service_instance, error_queue))
for thread in url_fetch_threads:
thread.start()
logging.info("Fetch URL Threads Started")
for thread in data_fetch_threads:
thread.start()
logging.info("Fetch Data Thread Started")
for thread in process_data_threads:
thread.start()
logging.info("Process Data Thread Started")
initial_backup_thread.join()
update_sheet_thread.start()
logging.info("Update Sheet Thread Started")
for thread in url_fetch_threads:
thread.join()
logging.info("Fetch URL Threads Completed")
# Ensure we only put None once per data fetch thread
for _ in data_fetch_threads:
url_fetch_queue.put(None)
for thread in data_fetch_threads:
thread.join()
logging.info("Fetch Data Thread Completed")
for _ in process_data_threads:
data_queue.put(None)
for thread in process_data_threads:
thread.join()
logging.info("Process Data Thread Completed")
update_sheet_thread.join()
logging.info("Update Sheet Thread Completed")
thread_completed.set()
logging.info("All Threads Completed")