File size: 7,239 Bytes
880c40b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
from bs4 import BeautifulSoup
import re
from queue import Queue
from .zzz_google_sheet_utils import push_to_sheet
import logging
import time
import threading

def regex_extract(strSoup):
    patterns = {
        "mvno": r"\[(.*?)\]",
        "plan_name": r"\]\s*(.*?)\s*\|",
        "monthly_fee": r"\|\s*([\d,]+์›)\s*\|",
        "monthly_data": r"(์›”\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ œํ•œ))",
        "daily_data": r"์›”\s*\d+\s*GB\s*\+.*?(๋งค์ผ\s*(?:[.\d]+(?:GB|MB)|๋ฌด์ œํ•œ))",
        "data_speed": r"\(([.\d]+(?:mbps|gbps))\)",
        "call_minutes": r"(\d+๋ถ„|๋ฌด์ œํ•œ)",
        "text_messages": r"(\d+๊ฑด|๋ฌด์ œํ•œ)",
        "carrier": r"(LG U\+|SKT|KT)",
        "network_type": r"(LTE|3G|4G|5G)",
        "discount_info": r"(\d+๊ฐœ์›”\s*์ดํ›„\s*[\d,]+์›)",
        "between_contract_and_call": r"(?<=ํ†ต์‹ ์‚ฌ ์•ฝ์ •)(.*?)(?=ํ†ตํ™”|ํŽผ์ณ๋ณด๊ธฐ)",
        "between_number_transfer_fee_and_sim_delivery": r"(?<=๋ฒˆํ˜ธ์ด๋™ ์ˆ˜์ˆ˜๋ฃŒ)(.*?)(?=์ผ๋ฐ˜ ์œ ์‹ฌ ๋ฐฐ์†ก)",
        "between_sim_delivery_and_nfc_sim": r"(?<=์ผ๋ฐ˜ ์œ ์‹ฌ ๋ฐฐ์†ก)(.*?)(?=NFC ์œ ์‹ฌ ๋ฐฐ์†ก)",
        "between_nfc_sim_and_esim": r"(?<=NFC ์œ ์‹ฌ ๋ฐฐ์†ก)(.*?)(?=eSIM)",
        "between_esim_and_support": r"(?<=eSIM)(.*?)(?=(์œ ์‹ฌ ๋ฐœ์†ก ํ›„ ์ทจ์†Œ ์‹œ|์ง€์›(?! ์•ˆํ•จ| ์•ˆ ํ•จ)))",
        "support_with_boundary": r'์ง€์›\s*(.*?)\s*๋ฏธ์ง€์›',
        "no_support_with_boundary": r'๋ฏธ์ง€์›\s*(.*?)\s*(์ ‘๊ธฐ|๊ธฐ๋ณธ)'
    }
    extracted = {key: re.search(pattern, strSoup) for key, pattern in patterns.items()}

    def format_extracted_categories(matches, categories):
        formatted = []
        for category in categories:
            for match in matches:
                if category in match:
                    start_index = match.find(category)
                    end_index = min([match.find(cat, start_index + 1) for cat in categories if cat in match[start_index + 1:]] + [len(match)])
                    additional_text = match[start_index + len(category):end_index].strip()
                    formatted_text = f"{category}: {additional_text}" if additional_text else category
                    formatted.append(formatted_text)
                    break
        return ', '.join(formatted)

    categories_support = ['๋ชจ๋ฐ”์ผ ํ•ซ์ŠคํŒŸ', '์†Œ์•ก ๊ฒฐ์ œ', 'ํ•ด์™ธ ๋กœ๋ฐ', '์ธํ„ฐ๋„ท ๊ฒฐํ•ฉ', '๋ฐ์ดํ„ฐ ์‰์–ด๋ง']
    categories_no_support = ['๋ชจ๋ฐ”์ผ ํ•ซ์ŠคํŒŸ', '์†Œ์•ก ๊ฒฐ์ œ', 'ํ•ด์™ธ ๋กœ๋ฐ', '์ธํ„ฐ๋„ท ๊ฒฐํ•ฉ', '๋ฐ์ดํ„ฐ ์‰์–ด๋ง']
    formatted_text_support = format_extracted_categories([extracted["support_with_boundary"].group(1) if extracted["support_with_boundary"] else ""], categories_support)
    formatted_text_no_support = format_extracted_categories([extracted["no_support_with_boundary"].group(1) if extracted["no_support_with_boundary"] else ""], categories_no_support)
    ์นด๋“œ_ํ• ์ธ = r"์นด๋“œ ๊ฒฐํ•ฉ ํ• ์ธ\s*(.*?)ํ• ์ธ"
    ์นด๋“œ_ํ• ์ธ_์ •๋ณด = re.search(์นด๋“œ_ํ• ์ธ, strSoup)
    return [
        extracted["mvno"].group(1) if extracted["mvno"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["plan_name"].group(1) if extracted["plan_name"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["monthly_fee"].group(1) if extracted["monthly_fee"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["monthly_data"].group(1) if extracted["monthly_data"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["daily_data"].group(1) if extracted["daily_data"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["data_speed"].group(1) if extracted["data_speed"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["call_minutes"].group(1) if extracted["call_minutes"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["text_messages"].group(1) if extracted["text_messages"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["carrier"].group(1) if extracted["carrier"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["network_type"].group(1) if extracted["network_type"] else "์ œ๊ณต์•ˆํ•จ", 
        extracted["discount_info"].group(1) if extracted["discount_info"] else "์ œ๊ณต์•ˆํ•จ",
        extracted["between_contract_and_call"].group(1) if extracted["between_contract_and_call"] else "์ œ๊ณต์•ˆํ•จ",
        extracted["between_number_transfer_fee_and_sim_delivery"].group(1) if extracted["between_number_transfer_fee_and_sim_delivery"] else "์ œ๊ณต์•ˆํ•จ",
        extracted["between_sim_delivery_and_nfc_sim"].group(1) if extracted["between_sim_delivery_and_nfc_sim"] else "์ œ๊ณต์•ˆํ•จ",
        extracted["between_nfc_sim_and_esim"].group(1) if extracted["between_nfc_sim_and_esim"] else "์ œ๊ณต์•ˆํ•จ",
        extracted["between_esim_and_support"].group(1) if extracted["between_esim_and_support"] else "์ œ๊ณต์•ˆํ•จ",
        formatted_text_support if formatted_text_support else "์ œ๊ณต์•ˆํ•จ",
        formatted_text_no_support if formatted_text_no_support else "์ œ๊ณต์•ˆํ•จ",
        "์ œ๊ณต์•ˆํ•จ",
        ์นด๋“œ_ํ• ์ธ_์ •๋ณด.group(1) + "ํ• ์ธ" if ์นด๋“œ_ํ• ์ธ_์ •๋ณด else "์ œ๊ณต์•ˆํ•จ", 
    ]

def process_data(data_queue, processed_data_queue, error_queue, num_threads):
    while True:
        item = data_queue.get()
        if item is None:
            break

        try:
            # sheet
            processed_data_queue.put(data)
            logging.info("Successfully processed data for URL")
        except Exception as e:

            error_queue.put("Error processing")

    logging.info("Finished processing data")

def convert_data_to_numeric(data):
    if isinstance(data, int) or isinstance(data, float):
        return data
    elif '๋ฌด์ œํ•œ' in data:
        return 100000
    elif data == '์ œ๊ณต์•ˆํ•จ':
        return 0
    else:
        data = data.replace('๋งค์ผ ', '')
        data = data.replace('๋งค์ผ', '')
        data = data.replace('์›” ', '')
        if 'GB' in data:
            return float(data.replace('GB', ''))
        elif 'MB' in data:
            return float(data.replace('MB', '')) / 1000
        else:
            return 0

def convert_calls_texts_to_numeric(data):
    if isinstance(data, int) or isinstance(data, float):
        return data
    elif data == '๋ฌด์ œํ•œ':
        return 100000
    elif data == '์ œ๊ณต์•ˆํ•จ':
        return 0
    elif isinstance(data, str):
        return int(data.replace('๋ถ„', '').replace('๊ฑด', ''))
    else:
        return 0

def calculate_score(rawMonthPayment, rawMonthData, rawDailyData, rawDataSpeed, rawCall, rawText):
    weights = {
        '์›” ์š”๊ธˆ': -1,
        '์›” ๋ฐ์ดํ„ฐ': 1,
        '์ผ ๋ฐ์ดํ„ฐ': 1,
        '๋ฐ์ดํ„ฐ ์†๋„': 1,
        'ํ†ตํ™”(๋ถ„)': 1,
        '๋ฌธ์ž(๊ฑด)': 1,
    }
    score = (weights['์›” ์š”๊ธˆ'] * rawMonthPayment +
             weights['์›” ๋ฐ์ดํ„ฐ'] * rawMonthData +
             weights['์ผ ๋ฐ์ดํ„ฐ'] * rawDailyData +
             weights['๋ฐ์ดํ„ฐ ์†๋„'] * rawDataSpeed +
             weights['ํ†ตํ™”(๋ถ„)'] * rawCall +
             weights['๋ฌธ์ž(๊ฑด)'] * rawText)
    return score

def retry_push_to_sheet(data, sheet_id, range, service_instance, backoff_factor=1):
    while True:
        try:
            result, service_instance = push_to_sheet(data, sheet_id, range, service_instance)
            return result
        except Exception as e:
            wait_time = backoff_factor
            print(f"Failed to push data to sheet: {e}. Retrying in {wait_time} seconds...")
            time.sleep(wait_time)