Sangwon78 commited on
Commit
6de0076
ยท
verified ยท
1 Parent(s): 14d364f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +46 -17
app.py CHANGED
@@ -6,7 +6,43 @@ from bs4 import BeautifulSoup
6
  import gradio as gr
7
  from urllib.parse import urlparse, urlunparse
8
 
9
- # ๋””๋ฒ„๊น… ๋กœ๊ทธ ์„ค์ • (UI์— ๋…ธ์ถœ๋˜์ง€ ์•Š๊ณ  debug.log ํŒŒ์ผ์— ๊ธฐ๋ก๋จ)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  logging.basicConfig(
11
  filename='debug.log',
12
  level=logging.DEBUG,
@@ -28,17 +64,15 @@ def get_random_user_agent():
28
 
29
  def convert_to_mobile_url(url):
30
  """
31
- blog.naver.com/ ๋ธ”๋กœ๊ทธ ์ฃผ์†Œ๋ฅผ m.blog.naver.com/ ํ˜•ํƒœ๋กœ ๋ณ€ํ™˜ํ•˜์—ฌ
32
- ๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€๋ฅผ ์š”์ฒญํ•˜๋„๋ก ํ•ด์ค๋‹ˆ๋‹ค.
33
  """
34
  parsed = urlparse(url)
35
- # PC ๋ฒ„์ „ ์ฃผ์†Œ์ธ ๊ฒฝ์šฐ -> m.blog.naver.com์œผ๋กœ ๋ณ€ํ™˜
36
  if "blog.naver.com" in parsed.netloc and "m.blog.naver.com" not in parsed.netloc:
37
  new_netloc = parsed.netloc.replace("blog.naver.com", "m.blog.naver.com")
38
  new_url = urlunparse((parsed.scheme, new_netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
39
  logging.debug(f"๋ชจ๋ฐ”์ผ ์ฃผ์†Œ๋กœ ๋ณ€ํ™˜: {new_url}")
40
  return new_url
41
- # ์ด๋ฏธ m.blog.naver.com์ธ ๊ฒฝ์šฐ ๋˜๋Š” ๋‹ค๋ฅธ ๋„๋ฉ”์ธ์ด๋ฉด ๊ทธ๋Œ€๋กœ ์‚ฌ์šฉ
42
  return url
43
 
44
  def scrape_naver_blog(url):
@@ -53,7 +87,7 @@ def scrape_naver_blog(url):
53
  logging.debug(f"์š”์ฒญ ์ „ {delay:.2f}์ดˆ ๋Œ€๊ธฐ.")
54
  time.sleep(delay)
55
 
56
- # 3) HTTP ํ—ค๋” ์„ค์ • (์‚ฌ๋žŒ์ฒ˜๋Ÿผ ๋ณด์ด๋„๋ก User-Agent, Referer ๋“ฑ ์„ค์ •)
57
  headers = {
58
  "User-Agent": get_random_user_agent(),
59
  "Referer": "https://m.blog.naver.com",
@@ -83,7 +117,7 @@ def scrape_naver_blog(url):
83
  # --------------------
84
  logging.debug("์ œ๋ชฉ ์ถ”์ถœ ์‹œ๋„.")
85
 
86
- # (A) ์ƒˆ ์—๋””ํ„ฐ ๊ตฌ์กฐ: div.se-module.se-module-text.se-title-text > p > span
87
  title_text = None
88
  title_div = soup.find("div", class_="se-module se-module-text se-title-text")
89
  if title_div:
@@ -93,18 +127,14 @@ def scrape_naver_blog(url):
93
  if span_tag:
94
  title_text = span_tag.get_text(strip=True)
95
 
96
- # (B) ๊ตฌ ์—๋””ํ„ฐ ๊ตฌ์กฐ: postViewArea ์•ˆ์˜ <strong>ํƒœ๊ทธ๋‚˜ <h3>ํƒœ๊ทธ, ๋˜๋Š” ๋‹ค๋ฅธ ์š”์†Œ์— ์ œ๋ชฉ์ด ์žˆ์„ ์ˆ˜ ์žˆ์Œ
97
  if not title_text:
98
- # postViewArea ๋‚ด๋ถ€์—์„œ ์ œ๋ชฉ์œผ๋กœ ๋ณด์ด๋Š” ํƒœ๊ทธ ํƒ์ƒ‰
99
  post_view_area = soup.find("div", id="postViewArea")
100
  if post_view_area:
101
- # ์˜ˆ์‹œ) <span class="pcol1 itemSubjectBoldfont">, <h3 class="se_textarea"> ๋“ฑ ๋‹ค์–‘
102
- # ์—ฌ๋Ÿฌ ์‹œ๋„๋ฅผ ํ•ด๋ด„
103
  candidate_title = post_view_area.find("span", class_="pcol1 itemSubjectBoldfont")
104
  if candidate_title:
105
  title_text = candidate_title.get_text(strip=True)
106
  else:
107
- # ๋‹ค๋ฅธ ํ˜•ํƒœ์˜ h3, strong, etc.
108
  h3_title = post_view_area.find("h3")
109
  if h3_title:
110
  title_text = h3_title.get_text(strip=True)
@@ -113,11 +143,10 @@ def scrape_naver_blog(url):
113
  if strong_title:
114
  title_text = strong_title.get_text(strip=True)
115
 
116
- # (C) ์œ„์—์„œ ๋ฐœ๊ฒฌํ•˜์ง€ ๋ชปํ–ˆ์œผ๋ฉด head > title์ด๋ผ๋„ ์‹œ๋„
117
  if not title_text:
118
  head_title = soup.find("title")
119
  if head_title:
120
- # "๋ธ”๋กœ๊ทธ ์ œ๋ชฉ : ๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ" ์ด๋Ÿฐ ์‹์ผ ์ˆ˜ ์žˆ์œผ๋‹ˆ ๊ทธ๋Œ€๋กœ ์‚ฌ์šฉ
121
  title_text = head_title.get_text(strip=True)
122
  else:
123
  title_text = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
@@ -141,18 +170,18 @@ def scrape_naver_blog(url):
141
  if old_content_div:
142
  content_text = old_content_div.get_text(separator="\n", strip=True)
143
 
144
- # (C) ๊ทธ๋ž˜๋„ ์—†์œผ๋ฉด ์•ˆ๋‚ด
145
  if not content_text:
146
  content_text = "๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
147
 
148
  logging.debug("๋ณธ๋ฌธ ๋‚ด์šฉ ์ถ”์ถœ ์™„๋ฃŒ.")
149
 
150
- # 6) ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ํ•˜๋‚˜์˜ ํ…์ŠคํŠธ๋กœ ๊ฒฐํ•ฉ
151
  combined_text = f"์ œ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}"
152
  logging.debug("์Šคํฌ๋ž˜ํ•‘ ํ”„๋กœ์„ธ์Šค ์™„๋ฃŒ.")
153
  return combined_text
154
 
155
- # Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ (UI์— ๋””๋ฒ„๊ทธ ๋กœ๊ทธ๋Š” ๋…ธ์ถœ๋˜์ง€ ์•Š์Œ)
156
  logging.debug("Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ.")
157
  iface = gr.Interface(
158
  fn=scrape_naver_blog,
 
6
  import gradio as gr
7
  from urllib.parse import urlparse, urlunparse
8
 
9
+ """
10
+ [๊ธฐ๋ณธ์‚ฌํ•ญ]
11
+ # ํ•œ๊ตญ์–ด๋กœ ์„ค๋ช…
12
+ # ํ—ˆ๊น…ํŽ˜์ด์Šค์˜ ๊ทธ๋ผ๋””์˜ค ํ™˜๊ฒฝ์—์„œ ๋™์ž‘๋˜๋Š” ์ŠคํŽ˜์ด์Šค
13
+ # ์š”์ฒญํ•œ ์‚ฌํ•ญ์˜ ์ฝ”๋“œ๋ฅผ ์ž„์˜์ ์œผ๋กœ ์ถ•์†Œ/์‚ญ์ œํ•˜์ง€ ๋ง๊ณ  ์ „์ฒด์ฝ”๋“œ๋ฅผ ์ถœ๋ ฅ
14
+
15
+ [์š”์ฒญ์‚ฌํ•ญ]
16
+ 1) ๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด, ํ•ด๋‹น ๋งํฌ์˜ ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์Šคํฌ๋ž˜ํ•‘
17
+ 2) ์ถœ๋ ฅ์€ ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ๋‚ด์šฉ์ด ํ…์ŠคํŠธ์ฐฝ์— ํ•œ ๋ฒˆ์— ๋‚˜์˜ค๋„๋ก
18
+ 3) ํƒ€๊ฒŸ URL ์˜ˆ์‹œ:
19
+ - ๊ธฐ๋ณธ: https://blog.naver.com/acoloje/223783254325
20
+ 4) ๋ฐ์ดํ„ฐ HTML ๊ตฌ์กฐ(์ œ๋ชฉ):
21
+ <div class="se-module se-module-text se-title-text">
22
+ <p class="se-text-paragraph se-text-paragraph-align-" style="" id="SE-d4c7e111-1d97-40cf-a490-eb188c4b7f04">
23
+ <span style="" class="se-fs- se-ff-" id="SE-b62ca798-fd52-11ef-8346-2d7ed694d63c"><!-- -->์†๋ฐ”๋‹ฅ์— M ๋งˆํฌ๊ฐ€ ์žˆ์–ด์š”?<!-- --></span>
24
+ </p>
25
+ </div>
26
+
27
+ 5) ๋ฐ์ดํ„ฐHTML ๊ตฌ์กฐ(๋ณธ๋ฌธ):
28
+ <div class="se-main-container">
29
+ ...
30
+ <div class="se-component se-text se-l-default" ...>
31
+ ...
32
+ <p class="se-text-paragraph ...">๋ณธ๋ฌธ ๋‚ด์šฉ</p>
33
+ ...
34
+ </div>
35
+ ...
36
+ </div>
37
+
38
+ [์ฐจ๋‹จ๋ฐฉ์ง€ ์ ์šฉ์ž๋ฃŒ]
39
+ 1. Header ์„ค์ •ํ•˜๊ธฐ (User-Agent, Referer ๋“ฑ)
40
+ 2. User-Agent ๋žœ๋ค ์„ค์ •
41
+ 3. Referer ์„ค์ • (๋„ค์ด๋ฒ„ ๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€์—์„œ ์˜จ ๊ฒƒ์ฒ˜๋Ÿผ ๋ณด์ด๋„๋ก)
42
+ 4. ๋žœ๋ค ๋”œ๋ ˆ์ด (1~3์ดˆ)
43
+ """
44
+
45
+ # ๋””๋ฒ„๊น… ๋กœ๊ทธ ์„ค์ • (UI์—๋Š” ๋ณด์ด์ง€ ์•Š๊ณ  debug.log ํŒŒ์ผ์— ๊ธฐ๋ก)
46
  logging.basicConfig(
47
  filename='debug.log',
48
  level=logging.DEBUG,
 
64
 
65
  def convert_to_mobile_url(url):
66
  """
67
+ PC ๋ฒ„์ „ blog.naver.com ์ฃผ์†Œ๋ฅผ m.blog.naver.com์œผ๋กœ ๋ณ€ํ™˜ํ•˜์—ฌ
68
+ ๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€๋ฅผ ์š”์ฒญํ•  ์ˆ˜ ์žˆ๋„๋ก ํ•จ.
69
  """
70
  parsed = urlparse(url)
 
71
  if "blog.naver.com" in parsed.netloc and "m.blog.naver.com" not in parsed.netloc:
72
  new_netloc = parsed.netloc.replace("blog.naver.com", "m.blog.naver.com")
73
  new_url = urlunparse((parsed.scheme, new_netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
74
  logging.debug(f"๋ชจ๋ฐ”์ผ ์ฃผ์†Œ๋กœ ๋ณ€ํ™˜: {new_url}")
75
  return new_url
 
76
  return url
77
 
78
  def scrape_naver_blog(url):
 
87
  logging.debug(f"์š”์ฒญ ์ „ {delay:.2f}์ดˆ ๋Œ€๊ธฐ.")
88
  time.sleep(delay)
89
 
90
+ # 3) HTTP ํ—ค๋” ์„ค์ • (User-Agent, Referer ๋“ฑ)
91
  headers = {
92
  "User-Agent": get_random_user_agent(),
93
  "Referer": "https://m.blog.naver.com",
 
117
  # --------------------
118
  logging.debug("์ œ๋ชฉ ์ถ”์ถœ ์‹œ๋„.")
119
 
120
+ # (A) ์ƒˆ ์—๋””ํ„ฐ ๊ตฌ์กฐ
121
  title_text = None
122
  title_div = soup.find("div", class_="se-module se-module-text se-title-text")
123
  if title_div:
 
127
  if span_tag:
128
  title_text = span_tag.get_text(strip=True)
129
 
130
+ # (B) ๊ตฌ ์—๋””ํ„ฐ ๊ตฌ์กฐ: #postViewArea ๋‚ด๋ถ€ ๋“ฑ
131
  if not title_text:
 
132
  post_view_area = soup.find("div", id="postViewArea")
133
  if post_view_area:
 
 
134
  candidate_title = post_view_area.find("span", class_="pcol1 itemSubjectBoldfont")
135
  if candidate_title:
136
  title_text = candidate_title.get_text(strip=True)
137
  else:
 
138
  h3_title = post_view_area.find("h3")
139
  if h3_title:
140
  title_text = h3_title.get_text(strip=True)
 
143
  if strong_title:
144
  title_text = strong_title.get_text(strip=True)
145
 
146
+ # (C) ๊ทธ๋ž˜๋„ ์—†์œผ๋ฉด <title>์ด๋ผ๋„ ์‚ฌ์šฉ
147
  if not title_text:
148
  head_title = soup.find("title")
149
  if head_title:
 
150
  title_text = head_title.get_text(strip=True)
151
  else:
152
  title_text = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
 
170
  if old_content_div:
171
  content_text = old_content_div.get_text(separator="\n", strip=True)
172
 
173
+ # (C) ์‹คํŒจ ์‹œ ์•ˆ๋‚ด
174
  if not content_text:
175
  content_text = "๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
176
 
177
  logging.debug("๋ณธ๋ฌธ ๋‚ด์šฉ ์ถ”์ถœ ์™„๋ฃŒ.")
178
 
179
+ # ์ œ๋ชฉ + ๋ณธ๋ฌธ ๊ฒฐํ•ฉ
180
  combined_text = f"์ œ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}"
181
  logging.debug("์Šคํฌ๋ž˜ํ•‘ ํ”„๋กœ์„ธ์Šค ์™„๋ฃŒ.")
182
  return combined_text
183
 
184
+ # Gradio ์ธํ„ฐํŽ˜์ด์Šค
185
  logging.debug("Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ.")
186
  iface = gr.Interface(
187
  fn=scrape_naver_blog,