moyo-crawling-requests / dev /processing.py
m8chaa's picture
λ―Έμ„±λ…„μž & 외ꡭ인 & 링크 - 자리 이동
fd73b84
Raw History Blame Contribute Delete
9.21 kB
import logging
import json
from .google_sheet_utils import push_to_sheet
import time
def process_data(data_queue, processed_data_queue, error_queue):
while True:
item = data_queue.get()
if item is None:
processed_data_queue.put(None)
data_queue.task_done()
break
try:
data_json = json.loads(item)
pageProps = data_json['props']['pageProps']
plan = pageProps['plan']
mvno = pageProps['mvno']
plan_meta = plan['meta']
plan_policy = plan['policy']
data = []
data.append("https://www.moyoplan.com/plans/" + str(plan_meta['id']))
data.append(mvno.get('mvnoName', ''))
data.append(plan_meta.get('name', ''))
data.append(str(plan_meta.get('feeString', '0')) + '원')
data.append(plan_meta.get('mobileDataString', '0'))
mobile_data2_str = plan_meta.get('mobileData2Str')
speed_when_exhausted_description = plan_meta.get('mobileDataDateExhaustedDescription')
data.append(mobile_data2_str if mobile_data2_str and mobile_data2_str != "μ—†μŒ" else 'μ œκ³΅μ•ˆν•¨')
data.append(speed_when_exhausted_description if speed_when_exhausted_description else 'μ œκ³΅μ•ˆν•¨')
mobile_voice = plan_meta.get('mobileVoice', 0)
mobile_message = plan_meta.get('mobileMessage', 0)
if isinstance(mobile_voice, int) and mobile_voice not in [-1, 0, 9999]:
mobile_voice = f"{mobile_voice:,}λΆ„"
elif mobile_voice in [-1, 0]:
mobile_voice = 'μ œκ³΅μ•ˆν•¨'
else:
mobile_voice = 'λ¬΄μ œν•œ'
if isinstance(mobile_message, int) and mobile_message not in [-1, 0, 9999]:
mobile_message = f"{mobile_message:,}건"
elif mobile_message in [-1, 0]:
mobile_message = 'μ œκ³΅μ•ˆν•¨'
else:
mobile_message = 'λ¬΄μ œν•œ'
data.append(mobile_voice)
data.append(mobile_message)
data.append(plan_meta.get('mno', ''))
data.append(plan_meta.get('net', ''))
discount_months = plan_meta.get('ν• μΈμ μš©_κ°œμ›”μˆ˜')
if discount_months is None:
data.append('μ œκ³΅μ•ˆν•¨')
else:
discount_months = str(discount_months) + 'κ°œμ›”'
discounted_price = plan_meta.get('ν• μΈλλ‚œν›„_κΈˆμ•‘')
if discounted_price is None:
discounted_price = 0
formatted_price = f"{discounted_price:,}원"
data.append(f"{discount_months} ν›„ {formatted_price}")
agreementType = plan_meta.get('agreementType', '')
agreementUnit = ''
if agreementType == 'month':
agreementUnit = 'κ°œμ›”'
elif agreementType == 'year':
agreementUnit = 'λ…„'
elif agreementType == 'day':
agreementUnit = '일'
data.append(f"{str(plan_meta.get('agreementPeriod') or 'μ—†μŒ')}" + agreementUnit)
if plan_policy is not None:
register_change_fee = plan_policy.get('registerChangeFee')
if register_change_fee is None:
register_change_fee = 0
else:
register_change_fee = f"{str(register_change_fee)}원"
else:
register_change_fee = 0
data.append('μ—†μŒ' if register_change_fee == 0 else register_change_fee)
data.append(plan.get('usimFeeDescription'))
data.append(plan.get('nfcDescription'))
data.append('유료(2,750원)' if plan.get('esimDescription') == '지원' else plan.get('esimDescription'))
supports = []
not_supports = []
if plan.get('esim', False):
supports.append('eSim')
else:
not_supports.append('eSim')
if plan.get('microPaymentDescription', '') == '지원':
supports.append('μ†Œμ•‘ 결제')
else:
not_supports.append('μ†Œμ•‘ 결제')
if plan.get('dataSharingDescription', '') == '지원':
supports.append('데이터 쉐어링')
else:
not_supports.append('데이터 쉐어링')
if plan.get('roamingDescription', '') == '지원':
supports.append('ν•΄μ™Έ 둜밍')
else:
not_supports.append('ν•΄μ™Έ 둜밍')
if plan.get('internetCombineDescription', '') == '지원':
supports.append('인터넷 κ²°ν•©')
else:
not_supports.append('인터넷 κ²°ν•©')
data.append('\n'.join(supports))
data.append('\n'.join(not_supports))
events = plan_meta.get('μ‚¬μ€ν’ˆλ¦¬μŠ€νŠΈ', [])
event_descriptions = []
event_links = []
for event in events:
event_descriptions.append(f"ν˜œνƒ: {event.get('description', '')}\nλŒ€μƒ: {event.get('condition', '')}\nμ§€κΈ‰μ‹œκΈ°: {event.get('receiptDate', '')}")
event_links.append(event.get('eventUrl', ''))
data.append('\n\n'.join(event_descriptions))
data.append('\n'.join(event_links))
card_discount = plan_meta.get('μΉ΄λ“œν˜œνƒμ μš©κ°€', '')
if card_discount:
data.append(f"30λ§Œμ› μ‚¬μš© μ‹œ μ›” {card_discount}원 할인")
else:
data.append('')
data.append(plan_meta.get('fee', 0))
data.append((plan_meta.get('mobileData') or 0) * 1000) # Convert GB to MB
data.append((plan_meta.get('mobileData2') or 0) * 1000) # Convert GB to MB
data.append((plan_meta.get('speedWhenExhausted') or 0)) # Convert Mbps to Kbps
data.append((plan_meta.get('mobileVoice') or 0))
data.append((plan_meta.get('mobileMessage') or 0))
try:
score = calculate_score(plan_meta.get('fee', 0), (plan_meta.get('mobileData') or 0) * 1000, (plan_meta.get('mobileData2') or 0) * 1000, (plan_meta.get('speedWhenExhausted') or 0), (plan_meta.get('mobileVoice') or 0), (plan_meta.get('mobileMessage') or 0))
except Exception as e:
logging.error(f"Error calculating score: {str(e)}")
score = 0
data.append(score)
data.append('')
data.append('')
signupForeignerCondition = ''
signupMinorCondition = ''
if plan_meta.get('policy'):
signupMinorCondition = plan_meta['policy'].get('signupMinorCondition', '')
signupForeignerCondition = plan_meta['policy'].get('signupForeignerCondition', '')
data.append(signupMinorCondition)
data.append(signupForeignerCondition)
else:
for policy in mvno['policy']:
mno = policy.get('mno', '')
signupMinorCondition = policy.get('signupMinorCondition', '')
signupForeignerCondition = policy.get('signupForeignerCondition', '')
if mno == plan_meta.get('mno', ''):
data.append(signupMinorCondition)
data.append(signupForeignerCondition)
break
else: # This else corresponds to the for loop, not the if statement
data.append('κ°€μž… λΆˆκ°€')
data.append('κ°€μž… λΆˆκ°€')
data.append((plan_meta.get('signupUrl', '') or ''))
data.append((plan_meta.get('selfSignupUrl', '') or ''))
processed_data_queue.put(data)
except Exception as e:
error_queue.put(f"Error processing data: {str(e)}")
logging.error(f"Exception in data processing: {e}")
finally:
data_queue.task_done()
logging.info("Finished processing data")
def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText):
weights = {
'μ›” μš”κΈˆ': -1,
'μ›” 데이터': 1,
'일 데이터': 1,
'데이터 속도': 1,
'톡화(λΆ„)': 1,
'문자(건)': 1,
}
score = (weights['μ›” μš”κΈˆ'] * rawMonthPayment +
weights['μ›” 데이터'] * rawMonthData +
weights['일 데이터'] * rawDailyData +
weights['데이터 속도'] * rawDataSpeed +
weights['톡화(λΆ„)'] * rawCall +
weights['문자(건)'] * rawText)
return score
def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1):
while True:
try:
result, service_instance = push_to_sheet(data, sheet_id, range, service_instance)
return result
except Exception as e:
wait_time = backoff_factor
logging.error(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...")
time.sleep(wait_time)