Download dev/processing.py from m8chaa/moyo-crawling-selenium-test: direct link, hf CLI and curl.
- Browser
- Download file 9.21 kB
-
https://huggingface.co/spaces/m8chaa/moyo-crawling-selenium-test/resolve/main/dev/processing.py
- Command line
-
hf download hf://spaces/m8chaa/moyo-crawling-selenium-test/dev/processing.py
-
curl -L -o processing.py https://huggingface.co/spaces/m8chaa/moyo-crawling-selenium-test/resolve/main/dev/processing.py
9.21 kB
| import logging | |
| import json | |
| from .google_sheet_utils import push_to_sheet | |
| import time | |
| def process_data(data_queue, processed_data_queue, error_queue): | |
| while True: | |
| item = data_queue.get() | |
| if item is None: | |
| processed_data_queue.put(None) | |
| data_queue.task_done() | |
| break | |
| try: | |
| data_json = json.loads(item) | |
| pageProps = data_json['props']['pageProps'] | |
| plan = pageProps['plan'] | |
| mvno = pageProps['mvno'] | |
| plan_meta = plan['meta'] | |
| plan_policy = plan['policy'] | |
| data = [] | |
| data.append("https://www.moyoplan.com/plans/" + str(plan_meta['id'])) | |
| data.append(mvno.get('mvnoName', '')) | |
| data.append(plan_meta.get('name', '')) | |
| data.append(str(plan_meta.get('feeString', '0')) + 'μ') | |
| data.append(plan_meta.get('mobileDataString', '0')) | |
| mobile_data2_str = plan_meta.get('mobileData2Str') | |
| speed_when_exhausted_description = plan_meta.get('mobileDataDateExhaustedDescription') | |
| data.append(mobile_data2_str if mobile_data2_str and mobile_data2_str != "μμ" else 'μ 곡μν¨') | |
| data.append(speed_when_exhausted_description if speed_when_exhausted_description else 'μ 곡μν¨') | |
| mobile_voice = plan_meta.get('mobileVoice', 0) | |
| mobile_message = plan_meta.get('mobileMessage', 0) | |
| if isinstance(mobile_voice, int) and mobile_voice not in [-1, 0, 9999]: | |
| mobile_voice = f"{mobile_voice:,}λΆ" | |
| elif mobile_voice in [-1, 0]: | |
| mobile_voice = 'μ 곡μν¨' | |
| else: | |
| mobile_voice = '무μ ν' | |
| if isinstance(mobile_message, int) and mobile_message not in [-1, 0, 9999]: | |
| mobile_message = f"{mobile_message:,}건" | |
| elif mobile_message in [-1, 0]: | |
| mobile_message = 'μ 곡μν¨' | |
| else: | |
| mobile_message = '무μ ν' | |
| data.append(mobile_voice) | |
| data.append(mobile_message) | |
| data.append(plan_meta.get('mno', '')) | |
| data.append(plan_meta.get('net', '')) | |
| discount_months = plan_meta.get('ν μΈμ μ©_κ°μμ') | |
| if discount_months is None: | |
| data.append('μ 곡μν¨') | |
| else: | |
| discount_months = str(discount_months) + 'κ°μ' | |
| discounted_price = plan_meta.get('ν μΈλλν_κΈμ‘') | |
| if discounted_price is None: | |
| discounted_price = 0 | |
| formatted_price = f"{discounted_price:,}μ" | |
| data.append(f"{discount_months} ν {formatted_price}") | |
| agreementType = plan_meta.get('agreementType', '') | |
| agreementUnit = '' | |
| if agreementType == 'month': | |
| agreementUnit = 'κ°μ' | |
| elif agreementType == 'year': | |
| agreementUnit = 'λ ' | |
| elif agreementType == 'day': | |
| agreementUnit = 'μΌ' | |
| data.append(f"{str(plan_meta.get('agreementPeriod') or 'μμ')}" + agreementUnit) | |
| if plan_policy is not None: | |
| register_change_fee = plan_policy.get('registerChangeFee') | |
| if register_change_fee is None: | |
| register_change_fee = 0 | |
| else: | |
| register_change_fee = f"{str(register_change_fee)}μ" | |
| else: | |
| register_change_fee = 0 | |
| data.append('μμ' if register_change_fee == 0 else register_change_fee) | |
| data.append(plan.get('usimFeeDescription')) | |
| data.append(plan.get('nfcDescription')) | |
| data.append('μ λ£(2,750μ)' if plan.get('esimDescription') == 'μ§μ' else plan.get('esimDescription')) | |
| supports = [] | |
| not_supports = [] | |
| if plan.get('esim', False): | |
| supports.append('eSim') | |
| else: | |
| not_supports.append('eSim') | |
| if plan.get('microPaymentDescription', '') == 'μ§μ': | |
| supports.append('μμ‘ κ²°μ ') | |
| else: | |
| not_supports.append('μμ‘ κ²°μ ') | |
| if plan.get('dataSharingDescription', '') == 'μ§μ': | |
| supports.append('λ°μ΄ν° μμ΄λ§') | |
| else: | |
| not_supports.append('λ°μ΄ν° μμ΄λ§') | |
| if plan.get('roamingDescription', '') == 'μ§μ': | |
| supports.append('ν΄μΈ λ‘λ°') | |
| else: | |
| not_supports.append('ν΄μΈ λ‘λ°') | |
| if plan.get('internetCombineDescription', '') == 'μ§μ': | |
| supports.append('μΈν°λ· κ²°ν©') | |
| else: | |
| not_supports.append('μΈν°λ· κ²°ν©') | |
| data.append('\n'.join(supports)) | |
| data.append('\n'.join(not_supports)) | |
| events = plan_meta.get('μ¬μν리μ€νΈ', []) | |
| event_descriptions = [] | |
| event_links = [] | |
| for event in events: | |
| event_descriptions.append(f"νν: {event.get('description', '')}\nλμ: {event.get('condition', '')}\nμ§κΈμκΈ°: {event.get('receiptDate', '')}") | |
| event_links.append(event.get('eventUrl', '')) | |
| data.append('\n\n'.join(event_descriptions)) | |
| data.append('\n'.join(event_links)) | |
| card_discount = plan_meta.get('μΉ΄λννμ μ©κ°', '') | |
| if card_discount: | |
| data.append(f"30λ§μ μ¬μ© μ μ {card_discount}μ ν μΈ") | |
| else: | |
| data.append('') | |
| data.append(plan_meta.get('fee', 0)) | |
| data.append((plan_meta.get('mobileData') or 0) * 1000) # Convert GB to MB | |
| data.append((plan_meta.get('mobileData2') or 0) * 1000) # Convert GB to MB | |
| data.append((plan_meta.get('speedWhenExhausted') or 0)) # Convert Mbps to Kbps | |
| data.append((plan_meta.get('mobileVoice') or 0)) | |
| data.append((plan_meta.get('mobileMessage') or 0)) | |
| try: | |
| score = calculate_score(plan_meta.get('fee', 0), (plan_meta.get('mobileData') or 0) * 1000, (plan_meta.get('mobileData2') or 0) * 1000, (plan_meta.get('speedWhenExhausted') or 0), (plan_meta.get('mobileVoice') or 0), (plan_meta.get('mobileMessage') or 0)) | |
| except Exception as e: | |
| logging.error(f"Error calculating score: {str(e)}") | |
| score = 0 | |
| data.append(score) | |
| data.append('') | |
| data.append('') | |
| signupForeignerCondition = '' | |
| signupMinorCondition = '' | |
| if plan_meta.get('policy'): | |
| signupMinorCondition = plan_meta['policy'].get('signupMinorCondition', '') | |
| signupForeignerCondition = plan_meta['policy'].get('signupForeignerCondition', '') | |
| data.append(signupMinorCondition) | |
| data.append(signupForeignerCondition) | |
| else: | |
| for policy in mvno['policy']: | |
| mno = policy.get('mno', '') | |
| signupMinorCondition = policy.get('signupMinorCondition', '') | |
| signupForeignerCondition = policy.get('signupForeignerCondition', '') | |
| if mno == plan_meta.get('mno', ''): | |
| data.append(signupMinorCondition) | |
| data.append(signupForeignerCondition) | |
| break | |
| else: # This else corresponds to the for loop, not the if statement | |
| data.append('κ°μ λΆκ°') | |
| data.append('κ°μ λΆκ°') | |
| data.append((plan_meta.get('signupUrl', '') or '')) | |
| data.append((plan_meta.get('selfSignupUrl', '') or '')) | |
| processed_data_queue.put(data) | |
| except Exception as e: | |
| error_queue.put(f"Error processing data: {str(e)}") | |
| logging.error(f"Exception in data processing: {e}") | |
| finally: | |
| data_queue.task_done() | |
| logging.info("Finished processing data") | |
| def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText): | |
| weights = { | |
| 'μ μκΈ': -1, | |
| 'μ λ°μ΄ν°': 1, | |
| 'μΌ λ°μ΄ν°': 1, | |
| 'λ°μ΄ν° μλ': 1, | |
| 'ν΅ν(λΆ)': 1, | |
| 'λ¬Έμ(건)': 1, | |
| } | |
| score = (weights['μ μκΈ'] * rawMonthPayment + | |
| weights['μ λ°μ΄ν°'] * rawMonthData + | |
| weights['μΌ λ°μ΄ν°'] * rawDailyData + | |
| weights['λ°μ΄ν° μλ'] * rawDataSpeed + | |
| weights['ν΅ν(λΆ)'] * rawCall + | |
| weights['λ¬Έμ(건)'] * rawText) | |
| return score | |
| def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1): | |
| while True: | |
| try: | |
| result, service_instance = push_to_sheet(data, sheet_id, range, service_instance) | |
| return result | |
| except Exception as e: | |
| wait_time = backoff_factor | |
| logging.error(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...") | |
| time.sleep(wait_time) | |