import logging import json from .google_sheet_utils import push_to_sheet import time def process_data(data_queue, processed_data_queue, error_queue): while True: item = data_queue.get() if item is None: processed_data_queue.put(None) data_queue.task_done() break try: data_json = json.loads(item) pageProps = data_json['props']['pageProps'] plan = pageProps['plan'] mvno = pageProps['mvno'] plan_meta = plan['meta'] plan_policy = plan['policy'] data = [] data.append("https://www.moyoplan.com/plans/" + str(plan_meta['id'])) data.append(mvno.get('mvnoName', '')) data.append(plan_meta.get('name', '')) data.append(str(plan_meta.get('feeString', '0')) + '원') data.append(plan_meta.get('mobileDataString', '0')) mobile_data2_str = plan_meta.get('mobileData2Str') speed_when_exhausted_description = plan_meta.get('mobileDataDateExhaustedDescription') data.append(mobile_data2_str if mobile_data2_str and mobile_data2_str != "없음" else '제공안함') data.append(speed_when_exhausted_description if speed_when_exhausted_description else '제공안함') mobile_voice = plan_meta.get('mobileVoice', 0) mobile_message = plan_meta.get('mobileMessage', 0) if isinstance(mobile_voice, int) and mobile_voice not in [-1, 0, 9999]: mobile_voice = f"{mobile_voice:,}분" elif mobile_voice in [-1, 0]: mobile_voice = '제공안함' else: mobile_voice = '무제한' if isinstance(mobile_message, int) and mobile_message not in [-1, 0, 9999]: mobile_message = f"{mobile_message:,}건" elif mobile_message in [-1, 0]: mobile_message = '제공안함' else: mobile_message = '무제한' data.append(mobile_voice) data.append(mobile_message) data.append(plan_meta.get('mno', '')) data.append(plan_meta.get('net', '')) discount_months = plan_meta.get('할인적용_개월수') if discount_months is None: data.append('제공안함') else: discount_months = str(discount_months) + '개월' discounted_price = plan_meta.get('할인끝난후_금액') if discounted_price is None: discounted_price = 0 formatted_price = f"{discounted_price:,}원" data.append(f"{discount_months} 후 {formatted_price}") agreementType = plan_meta.get('agreementType', '') agreementUnit = '' if agreementType == 'month': agreementUnit = '개월' elif agreementType == 'year': agreementUnit = '년' elif agreementType == 'day': agreementUnit = '일' data.append(f"{str(plan_meta.get('agreementPeriod') or '없음')}" + agreementUnit) if plan_policy is not None: register_change_fee = plan_policy.get('registerChangeFee') if register_change_fee is None: register_change_fee = 0 else: register_change_fee = f"{str(register_change_fee)}원" else: register_change_fee = 0 data.append('없음' if register_change_fee == 0 else register_change_fee) data.append(plan.get('usimFeeDescription')) data.append(plan.get('nfcDescription')) data.append('유료(2,750원)' if plan.get('esimDescription') == '지원' else plan.get('esimDescription')) supports = [] not_supports = [] if plan.get('esim', False): supports.append('eSim') else: not_supports.append('eSim') if plan.get('microPaymentDescription', '') == '지원': supports.append('소액 결제') else: not_supports.append('소액 결제') if plan.get('dataSharingDescription', '') == '지원': supports.append('데이터 쉐어링') else: not_supports.append('데이터 쉐어링') if plan.get('roamingDescription', '') == '지원': supports.append('해외 로밍') else: not_supports.append('해외 로밍') if plan.get('internetCombineDescription', '') == '지원': supports.append('인터넷 결합') else: not_supports.append('인터넷 결합') data.append('\n'.join(supports)) data.append('\n'.join(not_supports)) events = plan_meta.get('사은품리스트', []) event_descriptions = [] event_links = [] for event in events: event_descriptions.append(f"혜택: {event.get('description', '')}\n대상: {event.get('condition', '')}\n지급시기: {event.get('receiptDate', '')}") event_links.append(event.get('eventUrl', '')) data.append('\n\n'.join(event_descriptions)) data.append('\n'.join(event_links)) card_discount = plan_meta.get('카드혜택적용가', '') if card_discount: data.append(f"30만원 사용 시 월 {card_discount}원 할인") else: data.append('') data.append(plan_meta.get('fee', 0)) data.append((plan_meta.get('mobileData') or 0) * 1000) # Convert GB to MB data.append((plan_meta.get('mobileData2') or 0) * 1000) # Convert GB to MB data.append((plan_meta.get('speedWhenExhausted') or 0)) # Convert Mbps to Kbps data.append((plan_meta.get('mobileVoice') or 0)) data.append((plan_meta.get('mobileMessage') or 0)) try: score = calculate_score(plan_meta.get('fee', 0), (plan_meta.get('mobileData') or 0) * 1000, (plan_meta.get('mobileData2') or 0) * 1000, (plan_meta.get('speedWhenExhausted') or 0), (plan_meta.get('mobileVoice') or 0), (plan_meta.get('mobileMessage') or 0)) except Exception as e: logging.error(f"Error calculating score: {str(e)}") score = 0 data.append(score) data.append('') data.append('') signupForeignerCondition = '' signupMinorCondition = '' if plan_meta.get('policy'): signupMinorCondition = plan_meta['policy'].get('signupMinorCondition', '') signupForeignerCondition = plan_meta['policy'].get('signupForeignerCondition', '') data.append(signupMinorCondition) data.append(signupForeignerCondition) else: for policy in mvno['policy']: mno = policy.get('mno', '') signupMinorCondition = policy.get('signupMinorCondition', '') signupForeignerCondition = policy.get('signupForeignerCondition', '') if mno == plan_meta.get('mno', ''): data.append(signupMinorCondition) data.append(signupForeignerCondition) break else: # This else corresponds to the for loop, not the if statement data.append('가입 불가') data.append('가입 불가') data.append((plan_meta.get('signupUrl', '') or '')) data.append((plan_meta.get('selfSignupUrl', '') or '')) processed_data_queue.put(data) except Exception as e: error_queue.put(f"Error processing data: {str(e)}") logging.error(f"Exception in data processing: {e}") finally: data_queue.task_done() logging.info("Finished processing data") def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText): weights = { '월 요금': -1, '월 데이터': 1, '일 데이터': 1, '데이터 속도': 1, '통화(분)': 1, '문자(건)': 1, } score = (weights['월 요금'] * rawMonthPayment + weights['월 데이터'] * rawMonthData + weights['일 데이터'] * rawDailyData + weights['데이터 속도'] * rawDataSpeed + weights['통화(분)'] * rawCall + weights['문자(건)'] * rawText) return score def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1): while True: try: result, service_instance = push_to_sheet(data, sheet_id, range, service_instance) return result except Exception as e: wait_time = backoff_factor logging.error(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...") time.sleep(wait_time)