from bs4 import BeautifulSoup import re from queue import Queue from .zzz_google_sheet_utils import push_to_sheet import logging import time import threading def regex_extract(strSoup): patterns = { "mvno": r"\[(.*?)\]", "plan_name": r"\]\s*(.*?)\s*\|", "monthly_fee": r"\|\s*([\d,]+원)\s*\|", "monthly_data": r"(월\s*(?:[.\d]+(?:GB|MB)|무제한))", "daily_data": r"월\s*\d+\s*GB\s*\+.*?(매일\s*(?:[.\d]+(?:GB|MB)|무제한))", "data_speed": r"\(([.\d]+(?:mbps|gbps))\)", "call_minutes": r"(\d+분|무제한)", "text_messages": r"(\d+건|무제한)", "carrier": r"(LG U\+|SKT|KT)", "network_type": r"(LTE|3G|4G|5G)", "discount_info": r"(\d+개월\s*이후\s*[\d,]+원)", "between_contract_and_call": r"(?<=통신사 약정)(.*?)(?=통화|펼쳐보기)", "between_number_transfer_fee_and_sim_delivery": r"(?<=번호이동 수수료)(.*?)(?=일반 유심 배송)", "between_sim_delivery_and_nfc_sim": r"(?<=일반 유심 배송)(.*?)(?=NFC 유심 배송)", "between_nfc_sim_and_esim": r"(?<=NFC 유심 배송)(.*?)(?=eSIM)", "between_esim_and_support": r"(?<=eSIM)(.*?)(?=(유심 발송 후 취소 시|지원(?! 안함| 안 함)))", "support_with_boundary": r'지원\s*(.*?)\s*미지원', "no_support_with_boundary": r'미지원\s*(.*?)\s*(접기|기본)' } extracted = {key: re.search(pattern, strSoup) for key, pattern in patterns.items()} def format_extracted_categories(matches, categories): formatted = [] for category in categories: for match in matches: if category in match: start_index = match.find(category) end_index = min([match.find(cat, start_index + 1) for cat in categories if cat in match[start_index + 1:]] + [len(match)]) additional_text = match[start_index + len(category):end_index].strip() formatted_text = f"{category}: {additional_text}" if additional_text else category formatted.append(formatted_text) break return ', '.join(formatted) categories_support = ['모바일 핫스팟', '소액 결제', '해외 로밍', '인터넷 결합', '데이터 쉐어링'] categories_no_support = ['모바일 핫스팟', '소액 결제', '해외 로밍', '인터넷 결합', '데이터 쉐어링'] formatted_text_support = format_extracted_categories([extracted["support_with_boundary"].group(1) if extracted["support_with_boundary"] else ""], categories_support) formatted_text_no_support = format_extracted_categories([extracted["no_support_with_boundary"].group(1) if extracted["no_support_with_boundary"] else ""], categories_no_support) 카드_할인 = r"카드 결합 할인\s*(.*?)할인" 카드_할인_정보 = re.search(카드_할인, strSoup) return [ extracted["mvno"].group(1) if extracted["mvno"] else "제공안함", extracted["plan_name"].group(1) if extracted["plan_name"] else "제공안함", extracted["monthly_fee"].group(1) if extracted["monthly_fee"] else "제공안함", extracted["monthly_data"].group(1) if extracted["monthly_data"] else "제공안함", extracted["daily_data"].group(1) if extracted["daily_data"] else "제공안함", extracted["data_speed"].group(1) if extracted["data_speed"] else "제공안함", extracted["call_minutes"].group(1) if extracted["call_minutes"] else "제공안함", extracted["text_messages"].group(1) if extracted["text_messages"] else "제공안함", extracted["carrier"].group(1) if extracted["carrier"] else "제공안함", extracted["network_type"].group(1) if extracted["network_type"] else "제공안함", extracted["discount_info"].group(1) if extracted["discount_info"] else "제공안함", extracted["between_contract_and_call"].group(1) if extracted["between_contract_and_call"] else "제공안함", extracted["between_number_transfer_fee_and_sim_delivery"].group(1) if extracted["between_number_transfer_fee_and_sim_delivery"] else "제공안함", extracted["between_sim_delivery_and_nfc_sim"].group(1) if extracted["between_sim_delivery_and_nfc_sim"] else "제공안함", extracted["between_nfc_sim_and_esim"].group(1) if extracted["between_nfc_sim_and_esim"] else "제공안함", extracted["between_esim_and_support"].group(1) if extracted["between_esim_and_support"] else "제공안함", formatted_text_support if formatted_text_support else "제공안함", formatted_text_no_support if formatted_text_no_support else "제공안함", "제공안함", 카드_할인_정보.group(1) + "할인" if 카드_할인_정보 else "제공안함", ] def process_data(data_queue, processed_data_queue, error_queue, num_threads): while True: item = data_queue.get() if item is None: break try: # sheet processed_data_queue.put(data) logging.info("Successfully processed data for URL") except Exception as e: error_queue.put("Error processing") logging.info("Finished processing data") def convert_data_to_numeric(data): if isinstance(data, int) or isinstance(data, float): return data elif '무제한' in data: return 100000 elif data == '제공안함': return 0 else: data = data.replace('매일 ', '') data = data.replace('매일', '') data = data.replace('월 ', '') if 'GB' in data: return float(data.replace('GB', '')) elif 'MB' in data: return float(data.replace('MB', '')) / 1000 else: return 0 def convert_calls_texts_to_numeric(data): if isinstance(data, int) or isinstance(data, float): return data elif data == '무제한': return 100000 elif data == '제공안함': return 0 elif isinstance(data, str): return int(data.replace('분', '').replace('건', '')) else: return 0 def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText): weights = { '월 요금': -1, '월 데이터': 1, '일 데이터': 1, '데이터 속도': 1, '통화(분)': 1, '문자(건)': 1, } score = (weights['월 요금'] * rawMonthPayment + weights['월 데이터'] * rawMonthData + weights['일 데이터'] * rawDailyData + weights['데이터 속도'] * rawDataSpeed + weights['통화(분)'] * rawCall + weights['문자(건)'] * rawText) return score def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1): while True: try: result, service_instance = push_to_sheet(data, sheet_id, range, service_instance) return result except Exception as e: wait_time = backoff_factor print(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...") time.sleep(wait_time)