moyo-crawling-selenium-test / dev /zzz_processing.py
m8chaa's picture
request implsmnt
880c40b
Raw History Blame Contribute Delete
7.24 kB
from bs4 import BeautifulSoup
import re
from queue import Queue
from .zzz_google_sheet_utils import push_to_sheet
import logging
import time
import threading
def regex_extract(strSoup):
patterns = {
"mvno": r"\[(.*?)\]",
"plan_name": r"\]\s*(.*?)\s*\|",
"monthly_fee": r"\|\s*([\d,]+์›)\s*\|",
"monthly_data": r"(์›”\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ œํ•œ))",
"daily_data": r"์›”\s*\d+\s*GB\s*\+.*?(๋งค์ผ\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ œํ•œ))",
"data_speed": r"\(([.\d]+(?:mbps|gbps))\)",
"call_minutes": r"(\d+๋ถ„|๋ฌด์ œํ•œ)",
"text_messages": r"(\d+๊ฑด|๋ฌด์ œํ•œ)",
"carrier": r"(LG U\+|SKT|KT)",
"network_type": r"(LTE|3G|4G|5G)",
"discount_info": r"(\d+๊ฐœ์›”\s*์ดํ›„\s*[\d,]+์›)",
"between_contract_and_call": r"(?<=ํ†ต์‹ ์‚ฌ ์•ฝ์ •)(.*?)(?=ํ†ตํ™”|ํŽผ์ณ๋ณด๊ธฐ)",
"between_number_transfer_fee_and_sim_delivery": r"(?<=๋ฒˆํ˜ธ์ด๋™ ์ˆ˜์ˆ˜๋ฃŒ)(.*?)(?=์ผ๋ฐ˜ ์œ ์‹ฌ ๋ฐฐ์†ก)",
"between_sim_delivery_and_nfc_sim": r"(?<=์ผ๋ฐ˜ ์œ ์‹ฌ ๋ฐฐ์†ก)(.*?)(?=NFC ์œ ์‹ฌ ๋ฐฐ์†ก)",
"between_nfc_sim_and_esim": r"(?<=NFC ์œ ์‹ฌ ๋ฐฐ์†ก)(.*?)(?=eSIM)",
"between_esim_and_support": r"(?<=eSIM)(.*?)(?=(์œ ์‹ฌ ๋ฐœ์†ก ํ›„ ์ทจ์†Œ ์‹œ|์ง€์›(?! ์•ˆํ•จ| ์•ˆ ํ•จ)))",
"support_with_boundary": r'์ง€์›\s*(.*?)\s*๋ฏธ์ง€์›',
"no_support_with_boundary": r'๋ฏธ์ง€์›\s*(.*?)\s*(์ ‘๊ธฐ|๊ธฐ๋ณธ)'
}
extracted = {key: re.search(pattern, strSoup) for key, pattern in patterns.items()}
def format_extracted_categories(matches, categories):
formatted = []
for category in categories:
for match in matches:
if category in match:
start_index = match.find(category)
end_index = min([match.find(cat, start_index + 1) for cat in categories if cat in match[start_index + 1:]] + [len(match)])
additional_text = match[start_index + len(category):end_index].strip()
formatted_text = f"{category}: {additional_text}" if additional_text else category
formatted.append(formatted_text)
break
return ', '.join(formatted)
categories_support = ['๋ชจ๋ฐ”์ผ ํ•ซ์ŠคํŒŸ', '์†Œ์•ก ๊ฒฐ์ œ', 'ํ•ด์™ธ ๋กœ๋ฐ', '์ธํ„ฐ๋„ท ๊ฒฐํ•ฉ', '๋ฐ์ดํ„ฐ ์‰์–ด๋ง']
categories_no_support = ['๋ชจ๋ฐ”์ผ ํ•ซ์ŠคํŒŸ', '์†Œ์•ก ๊ฒฐ์ œ', 'ํ•ด์™ธ ๋กœ๋ฐ', '์ธํ„ฐ๋„ท ๊ฒฐํ•ฉ', '๋ฐ์ดํ„ฐ ์‰์–ด๋ง']
formatted_text_support = format_extracted_categories([extracted["support_with_boundary"].group(1) if extracted["support_with_boundary"] else ""], categories_support)
formatted_text_no_support = format_extracted_categories([extracted["no_support_with_boundary"].group(1) if extracted["no_support_with_boundary"] else ""], categories_no_support)
์นด๋“œ_ํ• ์ธ = r"์นด๋“œ ๊ฒฐํ•ฉ ํ• ์ธ\s*(.*?)ํ• ์ธ"
์นด๋“œ_ํ• ์ธ_์ •๋ณด = re.search(์นด๋“œ_ํ• ์ธ, strSoup)
return [
extracted["mvno"].group(1) if extracted["mvno"] else "์ œ๊ณต์•ˆํ•จ",
extracted["plan_name"].group(1) if extracted["plan_name"] else "์ œ๊ณต์•ˆํ•จ",
extracted["monthly_fee"].group(1) if extracted["monthly_fee"] else "์ œ๊ณต์•ˆํ•จ",
extracted["monthly_data"].group(1) if extracted["monthly_data"] else "์ œ๊ณต์•ˆํ•จ",
extracted["daily_data"].group(1) if extracted["daily_data"] else "์ œ๊ณต์•ˆํ•จ",
extracted["data_speed"].group(1) if extracted["data_speed"] else "์ œ๊ณต์•ˆํ•จ",
extracted["call_minutes"].group(1) if extracted["call_minutes"] else "์ œ๊ณต์•ˆํ•จ",
extracted["text_messages"].group(1) if extracted["text_messages"] else "์ œ๊ณต์•ˆํ•จ",
extracted["carrier"].group(1) if extracted["carrier"] else "์ œ๊ณต์•ˆํ•จ",
extracted["network_type"].group(1) if extracted["network_type"] else "์ œ๊ณต์•ˆํ•จ",
extracted["discount_info"].group(1) if extracted["discount_info"] else "์ œ๊ณต์•ˆํ•จ",
extracted["between_contract_and_call"].group(1) if extracted["between_contract_and_call"] else "์ œ๊ณต์•ˆํ•จ",
extracted["between_number_transfer_fee_and_sim_delivery"].group(1) if extracted["between_number_transfer_fee_and_sim_delivery"] else "์ œ๊ณต์•ˆํ•จ",
extracted["between_sim_delivery_and_nfc_sim"].group(1) if extracted["between_sim_delivery_and_nfc_sim"] else "์ œ๊ณต์•ˆํ•จ",
extracted["between_nfc_sim_and_esim"].group(1) if extracted["between_nfc_sim_and_esim"] else "์ œ๊ณต์•ˆํ•จ",
extracted["between_esim_and_support"].group(1) if extracted["between_esim_and_support"] else "์ œ๊ณต์•ˆํ•จ",
formatted_text_support if formatted_text_support else "์ œ๊ณต์•ˆํ•จ",
formatted_text_no_support if formatted_text_no_support else "์ œ๊ณต์•ˆํ•จ",
"์ œ๊ณต์•ˆํ•จ",
์นด๋“œ_ํ• ์ธ_์ •๋ณด.group(1) + "ํ• ์ธ" if ์นด๋“œ_ํ• ์ธ_์ •๋ณด else "์ œ๊ณต์•ˆํ•จ",
]
def process_data(data_queue, processed_data_queue, error_queue, num_threads):
while True:
item = data_queue.get()
if item is None:
break
try:
# sheet
processed_data_queue.put(data)
logging.info("Successfully processed data for URL")
except Exception as e:
error_queue.put("Error processing")
logging.info("Finished processing data")
def convert_data_to_numeric(data):
if isinstance(data, int) or isinstance(data, float):
return data
elif '๋ฌด์ œํ•œ' in data:
return 100000
elif data == '์ œ๊ณต์•ˆํ•จ':
return 0
else:
data = data.replace('๋งค์ผ ', '')
data = data.replace('๋งค์ผ', '')
data = data.replace('์›” ', '')
if 'GB' in data:
return float(data.replace('GB', ''))
elif 'MB' in data:
return float(data.replace('MB', '')) / 1000
else:
return 0
def convert_calls_texts_to_numeric(data):
if isinstance(data, int) or isinstance(data, float):
return data
elif data == '๋ฌด์ œํ•œ':
return 100000
elif data == '์ œ๊ณต์•ˆํ•จ':
return 0
elif isinstance(data, str):
return int(data.replace('๋ถ„', '').replace('๊ฑด', ''))
else:
return 0
def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText):
weights = {
'์›” ์š”๊ธˆ': -1,
'์›” ๋ฐ์ดํ„ฐ': 1,
'์ผ ๋ฐ์ดํ„ฐ': 1,
'๋ฐ์ดํ„ฐ ์†๋„': 1,
'ํ†ตํ™”(๋ถ„)': 1,
'๋ฌธ์ž(๊ฑด)': 1,
}
score = (weights['์›” ์š”๊ธˆ'] * rawMonthPayment +
weights['์›” ๋ฐ์ดํ„ฐ'] * rawMonthData +
weights['์ผ ๋ฐ์ดํ„ฐ'] * rawDailyData +
weights['๋ฐ์ดํ„ฐ ์†๋„'] * rawDataSpeed +
weights['ํ†ตํ™”(๋ถ„)'] * rawCall +
weights['๋ฌธ์ž(๊ฑด)'] * rawText)
return score
def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1):
while True:
try:
result, service_instance = push_to_sheet(data, sheet_id, range, service_instance)
return result
except Exception as e:
wait_time = backoff_factor
print(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...")
time.sleep(wait_time)