Update app.py
Browse files
app.py
CHANGED
|
@@ -6,7 +6,43 @@ from bs4 import BeautifulSoup
|
|
| 6 |
import gradio as gr
|
| 7 |
from urllib.parse import urlparse, urlunparse
|
| 8 |
|
| 9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
logging.basicConfig(
|
| 11 |
filename='debug.log',
|
| 12 |
level=logging.DEBUG,
|
|
@@ -28,17 +64,15 @@ def get_random_user_agent():
|
|
| 28 |
|
| 29 |
def convert_to_mobile_url(url):
|
| 30 |
"""
|
| 31 |
-
blog.naver.com
|
| 32 |
-
๋ชจ๋ฐ์ผ ํ์ด์ง๋ฅผ ์์ฒญ
|
| 33 |
"""
|
| 34 |
parsed = urlparse(url)
|
| 35 |
-
# PC ๋ฒ์ ์ฃผ์์ธ ๊ฒฝ์ฐ -> m.blog.naver.com์ผ๋ก ๋ณํ
|
| 36 |
if "blog.naver.com" in parsed.netloc and "m.blog.naver.com" not in parsed.netloc:
|
| 37 |
new_netloc = parsed.netloc.replace("blog.naver.com", "m.blog.naver.com")
|
| 38 |
new_url = urlunparse((parsed.scheme, new_netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
|
| 39 |
logging.debug(f"๋ชจ๋ฐ์ผ ์ฃผ์๋ก ๋ณํ: {new_url}")
|
| 40 |
return new_url
|
| 41 |
-
# ์ด๋ฏธ m.blog.naver.com์ธ ๊ฒฝ์ฐ ๋๋ ๋ค๋ฅธ ๋๋ฉ์ธ์ด๋ฉด ๊ทธ๋๋ก ์ฌ์ฉ
|
| 42 |
return url
|
| 43 |
|
| 44 |
def scrape_naver_blog(url):
|
|
@@ -53,7 +87,7 @@ def scrape_naver_blog(url):
|
|
| 53 |
logging.debug(f"์์ฒญ ์ {delay:.2f}์ด ๋๊ธฐ.")
|
| 54 |
time.sleep(delay)
|
| 55 |
|
| 56 |
-
# 3) HTTP ํค๋ ์ค์ (
|
| 57 |
headers = {
|
| 58 |
"User-Agent": get_random_user_agent(),
|
| 59 |
"Referer": "https://m.blog.naver.com",
|
|
@@ -83,7 +117,7 @@ def scrape_naver_blog(url):
|
|
| 83 |
# --------------------
|
| 84 |
logging.debug("์ ๋ชฉ ์ถ์ถ ์๋.")
|
| 85 |
|
| 86 |
-
# (A) ์ ์๋ํฐ ๊ตฌ์กฐ
|
| 87 |
title_text = None
|
| 88 |
title_div = soup.find("div", class_="se-module se-module-text se-title-text")
|
| 89 |
if title_div:
|
|
@@ -93,18 +127,14 @@ def scrape_naver_blog(url):
|
|
| 93 |
if span_tag:
|
| 94 |
title_text = span_tag.get_text(strip=True)
|
| 95 |
|
| 96 |
-
# (B) ๊ตฌ ์๋ํฐ ๊ตฌ์กฐ: postViewArea
|
| 97 |
if not title_text:
|
| 98 |
-
# postViewArea ๋ด๋ถ์์ ์ ๋ชฉ์ผ๋ก ๋ณด์ด๋ ํ๊ทธ ํ์
|
| 99 |
post_view_area = soup.find("div", id="postViewArea")
|
| 100 |
if post_view_area:
|
| 101 |
-
# ์์) <span class="pcol1 itemSubjectBoldfont">, <h3 class="se_textarea"> ๋ฑ ๋ค์
|
| 102 |
-
# ์ฌ๋ฌ ์๋๋ฅผ ํด๋ด
|
| 103 |
candidate_title = post_view_area.find("span", class_="pcol1 itemSubjectBoldfont")
|
| 104 |
if candidate_title:
|
| 105 |
title_text = candidate_title.get_text(strip=True)
|
| 106 |
else:
|
| 107 |
-
# ๋ค๋ฅธ ํํ์ h3, strong, etc.
|
| 108 |
h3_title = post_view_area.find("h3")
|
| 109 |
if h3_title:
|
| 110 |
title_text = h3_title.get_text(strip=True)
|
|
@@ -113,11 +143,10 @@ def scrape_naver_blog(url):
|
|
| 113 |
if strong_title:
|
| 114 |
title_text = strong_title.get_text(strip=True)
|
| 115 |
|
| 116 |
-
# (C)
|
| 117 |
if not title_text:
|
| 118 |
head_title = soup.find("title")
|
| 119 |
if head_title:
|
| 120 |
-
# "๋ธ๋ก๊ทธ ์ ๋ชฉ : ๋ค์ด๋ฒ ๋ธ๋ก๊ทธ" ์ด๋ฐ ์์ผ ์ ์์ผ๋ ๊ทธ๋๋ก ์ฌ์ฉ
|
| 121 |
title_text = head_title.get_text(strip=True)
|
| 122 |
else:
|
| 123 |
title_text = "์ ๋ชฉ์ ์ฐพ์ ์ ์์ต๋๋ค."
|
|
@@ -141,18 +170,18 @@ def scrape_naver_blog(url):
|
|
| 141 |
if old_content_div:
|
| 142 |
content_text = old_content_div.get_text(separator="\n", strip=True)
|
| 143 |
|
| 144 |
-
# (C)
|
| 145 |
if not content_text:
|
| 146 |
content_text = "๋ณธ๋ฌธ ๋ด์ฉ์ ์ฐพ์ ์ ์์ต๋๋ค."
|
| 147 |
|
| 148 |
logging.debug("๋ณธ๋ฌธ ๋ด์ฉ ์ถ์ถ ์๋ฃ.")
|
| 149 |
|
| 150 |
-
#
|
| 151 |
combined_text = f"์ ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}"
|
| 152 |
logging.debug("์คํฌ๋ํ ํ๋ก์ธ์ค ์๋ฃ.")
|
| 153 |
return combined_text
|
| 154 |
|
| 155 |
-
# Gradio ์ธํฐํ์ด์ค
|
| 156 |
logging.debug("Gradio ์ธํฐํ์ด์ค ์์ฑ.")
|
| 157 |
iface = gr.Interface(
|
| 158 |
fn=scrape_naver_blog,
|
|
|
|
| 6 |
import gradio as gr
|
| 7 |
from urllib.parse import urlparse, urlunparse
|
| 8 |
|
| 9 |
+
"""
|
| 10 |
+
[๊ธฐ๋ณธ์ฌํญ]
|
| 11 |
+
# ํ๊ตญ์ด๋ก ์ค๋ช
|
| 12 |
+
# ํ๊น
ํ์ด์ค์ ๊ทธ๋ผ๋์ค ํ๊ฒฝ์์ ๋์๋๋ ์คํ์ด์ค
|
| 13 |
+
# ์์ฒญํ ์ฌํญ์ ์ฝ๋๋ฅผ ์์์ ์ผ๋ก ์ถ์/์ญ์ ํ์ง ๋ง๊ณ ์ ์ฒด์ฝ๋๋ฅผ ์ถ๋ ฅ
|
| 14 |
+
|
| 15 |
+
[์์ฒญ์ฌํญ]
|
| 16 |
+
1) ๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ๋งํฌ๋ฅผ ์
๋ ฅํ๋ฉด, ํด๋น ๋งํฌ์ ์ ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋ด์ฉ์ ์คํฌ๋ํ
|
| 17 |
+
2) ์ถ๋ ฅ์ ์ ๋ชฉ๊ณผ ๋ณธ๋ฌธ๋ด์ฉ์ด ํ
์คํธ์ฐฝ์ ํ ๋ฒ์ ๋์ค๋๋ก
|
| 18 |
+
3) ํ๊ฒ URL ์์:
|
| 19 |
+
- ๊ธฐ๋ณธ: https://blog.naver.com/acoloje/223783254325
|
| 20 |
+
4) ๋ฐ์ดํฐ HTML ๊ตฌ์กฐ(์ ๋ชฉ):
|
| 21 |
+
<div class="se-module se-module-text se-title-text">
|
| 22 |
+
<p class="se-text-paragraph se-text-paragraph-align-" style="" id="SE-d4c7e111-1d97-40cf-a490-eb188c4b7f04">
|
| 23 |
+
<span style="" class="se-fs- se-ff-" id="SE-b62ca798-fd52-11ef-8346-2d7ed694d63c"><!-- -->์๋ฐ๋ฅ์ M ๋งํฌ๊ฐ ์์ด์?<!-- --></span>
|
| 24 |
+
</p>
|
| 25 |
+
</div>
|
| 26 |
+
|
| 27 |
+
5) ๋ฐ์ดํฐHTML ๊ตฌ์กฐ(๋ณธ๋ฌธ):
|
| 28 |
+
<div class="se-main-container">
|
| 29 |
+
...
|
| 30 |
+
<div class="se-component se-text se-l-default" ...>
|
| 31 |
+
...
|
| 32 |
+
<p class="se-text-paragraph ...">๋ณธ๋ฌธ ๋ด์ฉ</p>
|
| 33 |
+
...
|
| 34 |
+
</div>
|
| 35 |
+
...
|
| 36 |
+
</div>
|
| 37 |
+
|
| 38 |
+
[์ฐจ๋จ๋ฐฉ์ง ์ ์ฉ์๋ฃ]
|
| 39 |
+
1. Header ์ค์ ํ๊ธฐ (User-Agent, Referer ๋ฑ)
|
| 40 |
+
2. User-Agent ๋๋ค ์ค์
|
| 41 |
+
3. Referer ์ค์ (๋ค์ด๋ฒ ๋ชจ๋ฐ์ผ ํ์ด์ง์์ ์จ ๊ฒ์ฒ๋ผ ๋ณด์ด๋๋ก)
|
| 42 |
+
4. ๋๋ค ๋๋ ์ด (1~3์ด)
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
# ๋๋ฒ๊น
๋ก๊ทธ ์ค์ (UI์๋ ๋ณด์ด์ง ์๊ณ debug.log ํ์ผ์ ๊ธฐ๋ก)
|
| 46 |
logging.basicConfig(
|
| 47 |
filename='debug.log',
|
| 48 |
level=logging.DEBUG,
|
|
|
|
| 64 |
|
| 65 |
def convert_to_mobile_url(url):
|
| 66 |
"""
|
| 67 |
+
PC ๋ฒ์ blog.naver.com ์ฃผ์๋ฅผ m.blog.naver.com์ผ๋ก ๋ณํํ์ฌ
|
| 68 |
+
๋ชจ๋ฐ์ผ ํ์ด์ง๋ฅผ ์์ฒญํ ์ ์๋๋ก ํจ.
|
| 69 |
"""
|
| 70 |
parsed = urlparse(url)
|
|
|
|
| 71 |
if "blog.naver.com" in parsed.netloc and "m.blog.naver.com" not in parsed.netloc:
|
| 72 |
new_netloc = parsed.netloc.replace("blog.naver.com", "m.blog.naver.com")
|
| 73 |
new_url = urlunparse((parsed.scheme, new_netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
|
| 74 |
logging.debug(f"๋ชจ๋ฐ์ผ ์ฃผ์๋ก ๋ณํ: {new_url}")
|
| 75 |
return new_url
|
|
|
|
| 76 |
return url
|
| 77 |
|
| 78 |
def scrape_naver_blog(url):
|
|
|
|
| 87 |
logging.debug(f"์์ฒญ ์ {delay:.2f}์ด ๋๊ธฐ.")
|
| 88 |
time.sleep(delay)
|
| 89 |
|
| 90 |
+
# 3) HTTP ํค๋ ์ค์ (User-Agent, Referer ๋ฑ)
|
| 91 |
headers = {
|
| 92 |
"User-Agent": get_random_user_agent(),
|
| 93 |
"Referer": "https://m.blog.naver.com",
|
|
|
|
| 117 |
# --------------------
|
| 118 |
logging.debug("์ ๋ชฉ ์ถ์ถ ์๋.")
|
| 119 |
|
| 120 |
+
# (A) ์ ์๋ํฐ ๊ตฌ์กฐ
|
| 121 |
title_text = None
|
| 122 |
title_div = soup.find("div", class_="se-module se-module-text se-title-text")
|
| 123 |
if title_div:
|
|
|
|
| 127 |
if span_tag:
|
| 128 |
title_text = span_tag.get_text(strip=True)
|
| 129 |
|
| 130 |
+
# (B) ๊ตฌ ์๋ํฐ ๊ตฌ์กฐ: #postViewArea ๋ด๋ถ ๋ฑ
|
| 131 |
if not title_text:
|
|
|
|
| 132 |
post_view_area = soup.find("div", id="postViewArea")
|
| 133 |
if post_view_area:
|
|
|
|
|
|
|
| 134 |
candidate_title = post_view_area.find("span", class_="pcol1 itemSubjectBoldfont")
|
| 135 |
if candidate_title:
|
| 136 |
title_text = candidate_title.get_text(strip=True)
|
| 137 |
else:
|
|
|
|
| 138 |
h3_title = post_view_area.find("h3")
|
| 139 |
if h3_title:
|
| 140 |
title_text = h3_title.get_text(strip=True)
|
|
|
|
| 143 |
if strong_title:
|
| 144 |
title_text = strong_title.get_text(strip=True)
|
| 145 |
|
| 146 |
+
# (C) ๊ทธ๋๋ ์์ผ๋ฉด <title>์ด๋ผ๋ ์ฌ์ฉ
|
| 147 |
if not title_text:
|
| 148 |
head_title = soup.find("title")
|
| 149 |
if head_title:
|
|
|
|
| 150 |
title_text = head_title.get_text(strip=True)
|
| 151 |
else:
|
| 152 |
title_text = "์ ๋ชฉ์ ์ฐพ์ ์ ์์ต๋๋ค."
|
|
|
|
| 170 |
if old_content_div:
|
| 171 |
content_text = old_content_div.get_text(separator="\n", strip=True)
|
| 172 |
|
| 173 |
+
# (C) ์คํจ ์ ์๋ด
|
| 174 |
if not content_text:
|
| 175 |
content_text = "๋ณธ๋ฌธ ๋ด์ฉ์ ์ฐพ์ ์ ์์ต๋๋ค."
|
| 176 |
|
| 177 |
logging.debug("๋ณธ๋ฌธ ๋ด์ฉ ์ถ์ถ ์๋ฃ.")
|
| 178 |
|
| 179 |
+
# ์ ๋ชฉ + ๋ณธ๋ฌธ ๊ฒฐํฉ
|
| 180 |
combined_text = f"์ ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}"
|
| 181 |
logging.debug("์คํฌ๋ํ ํ๋ก์ธ์ค ์๋ฃ.")
|
| 182 |
return combined_text
|
| 183 |
|
| 184 |
+
# Gradio ์ธํฐํ์ด์ค
|
| 185 |
logging.debug("Gradio ์ธํฐํ์ด์ค ์์ฑ.")
|
| 186 |
iface = gr.Interface(
|
| 187 |
fn=scrape_naver_blog,
|