File size: 7,239 Bytes
880c40b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | from bs4 import BeautifulSoup
import re
from queue import Queue
from .zzz_google_sheet_utils import push_to_sheet
import logging
import time
import threading
def regex_extract(strSoup):
patterns = {
"mvno": r"\[(.*?)\]",
"plan_name": r"\]\s*(.*?)\s*\|",
"monthly_fee": r"\|\s*([\d,]+์)\s*\|",
"monthly_data": r"(์\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ ํ))",
"daily_data": r"์\s*\d+\s*GB\s*\+.*?(๋งค์ผ\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ ํ))",
"data_speed": r"\(([.\d]+(?:mbps|gbps))\)",
"call_minutes": r"(\d+๋ถ|๋ฌด์ ํ)",
"text_messages": r"(\d+๊ฑด|๋ฌด์ ํ)",
"carrier": r"(LG U\+|SKT|KT)",
"network_type": r"(LTE|3G|4G|5G)",
"discount_info": r"(\d+๊ฐ์\s*์ดํ\s*[\d,]+์)",
"between_contract_and_call": r"(?<=ํต์ ์ฌ ์ฝ์ )(.*?)(?=ํตํ|ํผ์ณ๋ณด๊ธฐ)",
"between_number_transfer_fee_and_sim_delivery": r"(?<=๋ฒํธ์ด๋ ์์๋ฃ)(.*?)(?=์ผ๋ฐ ์ ์ฌ ๋ฐฐ์ก)",
"between_sim_delivery_and_nfc_sim": r"(?<=์ผ๋ฐ ์ ์ฌ ๋ฐฐ์ก)(.*?)(?=NFC ์ ์ฌ ๋ฐฐ์ก)",
"between_nfc_sim_and_esim": r"(?<=NFC ์ ์ฌ ๋ฐฐ์ก)(.*?)(?=eSIM)",
"between_esim_and_support": r"(?<=eSIM)(.*?)(?=(์ ์ฌ ๋ฐ์ก ํ ์ทจ์ ์|์ง์(?! ์ํจ| ์ ํจ)))",
"support_with_boundary": r'์ง์\s*(.*?)\s*๋ฏธ์ง์',
"no_support_with_boundary": r'๋ฏธ์ง์\s*(.*?)\s*(์ ๊ธฐ|๊ธฐ๋ณธ)'
}
extracted = {key: re.search(pattern, strSoup) for key, pattern in patterns.items()}
def format_extracted_categories(matches, categories):
formatted = []
for category in categories:
for match in matches:
if category in match:
start_index = match.find(category)
end_index = min([match.find(cat, start_index + 1) for cat in categories if cat in match[start_index + 1:]] + [len(match)])
additional_text = match[start_index + len(category):end_index].strip()
formatted_text = f"{category}: {additional_text}" if additional_text else category
formatted.append(formatted_text)
break
return ', '.join(formatted)
categories_support = ['๋ชจ๋ฐ์ผ ํซ์คํ', '์์ก ๊ฒฐ์ ', 'ํด์ธ ๋ก๋ฐ', '์ธํฐ๋ท ๊ฒฐํฉ', '๋ฐ์ดํฐ ์์ด๋ง']
categories_no_support = ['๋ชจ๋ฐ์ผ ํซ์คํ', '์์ก ๊ฒฐ์ ', 'ํด์ธ ๋ก๋ฐ', '์ธํฐ๋ท ๊ฒฐํฉ', '๋ฐ์ดํฐ ์์ด๋ง']
formatted_text_support = format_extracted_categories([extracted["support_with_boundary"].group(1) if extracted["support_with_boundary"] else ""], categories_support)
formatted_text_no_support = format_extracted_categories([extracted["no_support_with_boundary"].group(1) if extracted["no_support_with_boundary"] else ""], categories_no_support)
์นด๋_ํ ์ธ = r"์นด๋ ๊ฒฐํฉ ํ ์ธ\s*(.*?)ํ ์ธ"
์นด๋_ํ ์ธ_์ ๋ณด = re.search(์นด๋_ํ ์ธ, strSoup)
return [
extracted["mvno"].group(1) if extracted["mvno"] else "์ ๊ณต์ํจ",
extracted["plan_name"].group(1) if extracted["plan_name"] else "์ ๊ณต์ํจ",
extracted["monthly_fee"].group(1) if extracted["monthly_fee"] else "์ ๊ณต์ํจ",
extracted["monthly_data"].group(1) if extracted["monthly_data"] else "์ ๊ณต์ํจ",
extracted["daily_data"].group(1) if extracted["daily_data"] else "์ ๊ณต์ํจ",
extracted["data_speed"].group(1) if extracted["data_speed"] else "์ ๊ณต์ํจ",
extracted["call_minutes"].group(1) if extracted["call_minutes"] else "์ ๊ณต์ํจ",
extracted["text_messages"].group(1) if extracted["text_messages"] else "์ ๊ณต์ํจ",
extracted["carrier"].group(1) if extracted["carrier"] else "์ ๊ณต์ํจ",
extracted["network_type"].group(1) if extracted["network_type"] else "์ ๊ณต์ํจ",
extracted["discount_info"].group(1) if extracted["discount_info"] else "์ ๊ณต์ํจ",
extracted["between_contract_and_call"].group(1) if extracted["between_contract_and_call"] else "์ ๊ณต์ํจ",
extracted["between_number_transfer_fee_and_sim_delivery"].group(1) if extracted["between_number_transfer_fee_and_sim_delivery"] else "์ ๊ณต์ํจ",
extracted["between_sim_delivery_and_nfc_sim"].group(1) if extracted["between_sim_delivery_and_nfc_sim"] else "์ ๊ณต์ํจ",
extracted["between_nfc_sim_and_esim"].group(1) if extracted["between_nfc_sim_and_esim"] else "์ ๊ณต์ํจ",
extracted["between_esim_and_support"].group(1) if extracted["between_esim_and_support"] else "์ ๊ณต์ํจ",
formatted_text_support if formatted_text_support else "์ ๊ณต์ํจ",
formatted_text_no_support if formatted_text_no_support else "์ ๊ณต์ํจ",
"์ ๊ณต์ํจ",
์นด๋_ํ ์ธ_์ ๋ณด.group(1) + "ํ ์ธ" if ์นด๋_ํ ์ธ_์ ๋ณด else "์ ๊ณต์ํจ",
]
def process_data(data_queue, processed_data_queue, error_queue, num_threads):
while True:
item = data_queue.get()
if item is None:
break
try:
# sheet
processed_data_queue.put(data)
logging.info("Successfully processed data for URL")
except Exception as e:
error_queue.put("Error processing")
logging.info("Finished processing data")
def convert_data_to_numeric(data):
if isinstance(data, int) or isinstance(data, float):
return data
elif '๋ฌด์ ํ' in data:
return 100000
elif data == '์ ๊ณต์ํจ':
return 0
else:
data = data.replace('๋งค์ผ ', '')
data = data.replace('๋งค์ผ', '')
data = data.replace('์ ', '')
if 'GB' in data:
return float(data.replace('GB', ''))
elif 'MB' in data:
return float(data.replace('MB', '')) / 1000
else:
return 0
def convert_calls_texts_to_numeric(data):
if isinstance(data, int) or isinstance(data, float):
return data
elif data == '๋ฌด์ ํ':
return 100000
elif data == '์ ๊ณต์ํจ':
return 0
elif isinstance(data, str):
return int(data.replace('๋ถ', '').replace('๊ฑด', ''))
else:
return 0
def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText):
weights = {
'์ ์๊ธ': -1,
'์ ๋ฐ์ดํฐ': 1,
'์ผ ๋ฐ์ดํฐ': 1,
'๋ฐ์ดํฐ ์๋': 1,
'ํตํ(๋ถ)': 1,
'๋ฌธ์(๊ฑด)': 1,
}
score = (weights['์ ์๊ธ'] * rawMonthPayment +
weights['์ ๋ฐ์ดํฐ'] * rawMonthData +
weights['์ผ ๋ฐ์ดํฐ'] * rawDailyData +
weights['๋ฐ์ดํฐ ์๋'] * rawDataSpeed +
weights['ํตํ(๋ถ)'] * rawCall +
weights['๋ฌธ์(๊ฑด)'] * rawText)
return score
def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1):
while True:
try:
result, service_instance = push_to_sheet(data, sheet_id, range, service_instance)
return result
except Exception as e:
wait_time = backoff_factor
print(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...")
time.sleep(wait_time)
|