Download dev/zzz_processing.py from m8chaa/moyo-crawling-selenium-test: direct link, hf CLI and curl.
- Browser
- Download file 7.24 kB
-
https://huggingface.co/spaces/m8chaa/moyo-crawling-selenium-test/resolve/main/dev/zzz_processing.py
- Command line
-
hf download hf://spaces/m8chaa/moyo-crawling-selenium-test/dev/zzz_processing.py
-
curl -L -o zzz_processing.py https://huggingface.co/spaces/m8chaa/moyo-crawling-selenium-test/resolve/main/dev/zzz_processing.py
7.24 kB
| from bs4 import BeautifulSoup | |
| import re | |
| from queue import Queue | |
| from .zzz_google_sheet_utils import push_to_sheet | |
| import logging | |
| import time | |
| import threading | |
| def regex_extract(strSoup): | |
| patterns = { | |
| "mvno": r"\[(.*?)\]", | |
| "plan_name": r"\]\s*(.*?)\s*\|", | |
| "monthly_fee": r"\|\s*([\d,]+์)\s*\|", | |
| "monthly_data": r"(์\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ ํ))", | |
| "daily_data": r"์\s*\d+\s*GB\s*\+.*?(๋งค์ผ\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ ํ))", | |
| "data_speed": r"\(([.\d]+(?:mbps|gbps))\)", | |
| "call_minutes": r"(\d+๋ถ|๋ฌด์ ํ)", | |
| "text_messages": r"(\d+๊ฑด|๋ฌด์ ํ)", | |
| "carrier": r"(LG U\+|SKT|KT)", | |
| "network_type": r"(LTE|3G|4G|5G)", | |
| "discount_info": r"(\d+๊ฐ์\s*์ดํ\s*[\d,]+์)", | |
| "between_contract_and_call": r"(?<=ํต์ ์ฌ ์ฝ์ )(.*?)(?=ํตํ|ํผ์ณ๋ณด๊ธฐ)", | |
| "between_number_transfer_fee_and_sim_delivery": r"(?<=๋ฒํธ์ด๋ ์์๋ฃ)(.*?)(?=์ผ๋ฐ ์ ์ฌ ๋ฐฐ์ก)", | |
| "between_sim_delivery_and_nfc_sim": r"(?<=์ผ๋ฐ ์ ์ฌ ๋ฐฐ์ก)(.*?)(?=NFC ์ ์ฌ ๋ฐฐ์ก)", | |
| "between_nfc_sim_and_esim": r"(?<=NFC ์ ์ฌ ๋ฐฐ์ก)(.*?)(?=eSIM)", | |
| "between_esim_and_support": r"(?<=eSIM)(.*?)(?=(์ ์ฌ ๋ฐ์ก ํ ์ทจ์ ์|์ง์(?! ์ํจ| ์ ํจ)))", | |
| "support_with_boundary": r'์ง์\s*(.*?)\s*๋ฏธ์ง์', | |
| "no_support_with_boundary": r'๋ฏธ์ง์\s*(.*?)\s*(์ ๊ธฐ|๊ธฐ๋ณธ)' | |
| } | |
| extracted = {key: re.search(pattern, strSoup) for key, pattern in patterns.items()} | |
| def format_extracted_categories(matches, categories): | |
| formatted = [] | |
| for category in categories: | |
| for match in matches: | |
| if category in match: | |
| start_index = match.find(category) | |
| end_index = min([match.find(cat, start_index + 1) for cat in categories if cat in match[start_index + 1:]] + [len(match)]) | |
| additional_text = match[start_index + len(category):end_index].strip() | |
| formatted_text = f"{category}: {additional_text}" if additional_text else category | |
| formatted.append(formatted_text) | |
| break | |
| return ', '.join(formatted) | |
| categories_support = ['๋ชจ๋ฐ์ผ ํซ์คํ', '์์ก ๊ฒฐ์ ', 'ํด์ธ ๋ก๋ฐ', '์ธํฐ๋ท ๊ฒฐํฉ', '๋ฐ์ดํฐ ์์ด๋ง'] | |
| categories_no_support = ['๋ชจ๋ฐ์ผ ํซ์คํ', '์์ก ๊ฒฐ์ ', 'ํด์ธ ๋ก๋ฐ', '์ธํฐ๋ท ๊ฒฐํฉ', '๋ฐ์ดํฐ ์์ด๋ง'] | |
| formatted_text_support = format_extracted_categories([extracted["support_with_boundary"].group(1) if extracted["support_with_boundary"] else ""], categories_support) | |
| formatted_text_no_support = format_extracted_categories([extracted["no_support_with_boundary"].group(1) if extracted["no_support_with_boundary"] else ""], categories_no_support) | |
| ์นด๋_ํ ์ธ = r"์นด๋ ๊ฒฐํฉ ํ ์ธ\s*(.*?)ํ ์ธ" | |
| ์นด๋_ํ ์ธ_์ ๋ณด = re.search(์นด๋_ํ ์ธ, strSoup) | |
| return [ | |
| extracted["mvno"].group(1) if extracted["mvno"] else "์ ๊ณต์ํจ", | |
| extracted["plan_name"].group(1) if extracted["plan_name"] else "์ ๊ณต์ํจ", | |
| extracted["monthly_fee"].group(1) if extracted["monthly_fee"] else "์ ๊ณต์ํจ", | |
| extracted["monthly_data"].group(1) if extracted["monthly_data"] else "์ ๊ณต์ํจ", | |
| extracted["daily_data"].group(1) if extracted["daily_data"] else "์ ๊ณต์ํจ", | |
| extracted["data_speed"].group(1) if extracted["data_speed"] else "์ ๊ณต์ํจ", | |
| extracted["call_minutes"].group(1) if extracted["call_minutes"] else "์ ๊ณต์ํจ", | |
| extracted["text_messages"].group(1) if extracted["text_messages"] else "์ ๊ณต์ํจ", | |
| extracted["carrier"].group(1) if extracted["carrier"] else "์ ๊ณต์ํจ", | |
| extracted["network_type"].group(1) if extracted["network_type"] else "์ ๊ณต์ํจ", | |
| extracted["discount_info"].group(1) if extracted["discount_info"] else "์ ๊ณต์ํจ", | |
| extracted["between_contract_and_call"].group(1) if extracted["between_contract_and_call"] else "์ ๊ณต์ํจ", | |
| extracted["between_number_transfer_fee_and_sim_delivery"].group(1) if extracted["between_number_transfer_fee_and_sim_delivery"] else "์ ๊ณต์ํจ", | |
| extracted["between_sim_delivery_and_nfc_sim"].group(1) if extracted["between_sim_delivery_and_nfc_sim"] else "์ ๊ณต์ํจ", | |
| extracted["between_nfc_sim_and_esim"].group(1) if extracted["between_nfc_sim_and_esim"] else "์ ๊ณต์ํจ", | |
| extracted["between_esim_and_support"].group(1) if extracted["between_esim_and_support"] else "์ ๊ณต์ํจ", | |
| formatted_text_support if formatted_text_support else "์ ๊ณต์ํจ", | |
| formatted_text_no_support if formatted_text_no_support else "์ ๊ณต์ํจ", | |
| "์ ๊ณต์ํจ", | |
| ์นด๋_ํ ์ธ_์ ๋ณด.group(1) + "ํ ์ธ" if ์นด๋_ํ ์ธ_์ ๋ณด else "์ ๊ณต์ํจ", | |
| ] | |
| def process_data(data_queue, processed_data_queue, error_queue, num_threads): | |
| while True: | |
| item = data_queue.get() | |
| if item is None: | |
| break | |
| try: | |
| # sheet | |
| processed_data_queue.put(data) | |
| logging.info("Successfully processed data for URL") | |
| except Exception as e: | |
| error_queue.put("Error processing") | |
| logging.info("Finished processing data") | |
| def convert_data_to_numeric(data): | |
| if isinstance(data, int) or isinstance(data, float): | |
| return data | |
| elif '๋ฌด์ ํ' in data: | |
| return 100000 | |
| elif data == '์ ๊ณต์ํจ': | |
| return 0 | |
| else: | |
| data = data.replace('๋งค์ผ ', '') | |
| data = data.replace('๋งค์ผ', '') | |
| data = data.replace('์ ', '') | |
| if 'GB' in data: | |
| return float(data.replace('GB', '')) | |
| elif 'MB' in data: | |
| return float(data.replace('MB', '')) / 1000 | |
| else: | |
| return 0 | |
| def convert_calls_texts_to_numeric(data): | |
| if isinstance(data, int) or isinstance(data, float): | |
| return data | |
| elif data == '๋ฌด์ ํ': | |
| return 100000 | |
| elif data == '์ ๊ณต์ํจ': | |
| return 0 | |
| elif isinstance(data, str): | |
| return int(data.replace('๋ถ', '').replace('๊ฑด', '')) | |
| else: | |
| return 0 | |
| def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText): | |
| weights = { | |
| '์ ์๊ธ': -1, | |
| '์ ๋ฐ์ดํฐ': 1, | |
| '์ผ ๋ฐ์ดํฐ': 1, | |
| '๋ฐ์ดํฐ ์๋': 1, | |
| 'ํตํ(๋ถ)': 1, | |
| '๋ฌธ์(๊ฑด)': 1, | |
| } | |
| score = (weights['์ ์๊ธ'] * rawMonthPayment + | |
| weights['์ ๋ฐ์ดํฐ'] * rawMonthData + | |
| weights['์ผ ๋ฐ์ดํฐ'] * rawDailyData + | |
| weights['๋ฐ์ดํฐ ์๋'] * rawDataSpeed + | |
| weights['ํตํ(๋ถ)'] * rawCall + | |
| weights['๋ฌธ์(๊ฑด)'] * rawText) | |
| return score | |
| def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1): | |
| while True: | |
| try: | |
| result, service_instance = push_to_sheet(data, sheet_id, range, service_instance) | |
| return result | |
| except Exception as e: | |
| wait_time = backoff_factor | |
| print(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...") | |
| time.sleep(wait_time) | |