Update app.py
Browse files
app.py
CHANGED
|
@@ -1,97 +1,86 @@
|
|
|
|
|
| 1 |
import requests
|
| 2 |
from bs4 import BeautifulSoup
|
| 3 |
-
import
|
| 4 |
-
import
|
| 5 |
-
import
|
| 6 |
-
import tempfile
|
| 7 |
|
| 8 |
-
#
|
| 9 |
-
|
| 10 |
-
# ํค์๋ ๊ธฐ๋ฐ ๋ค์ด๋ฒ ๋ด์ค ๊ฒ์ URL ์์ฑ (URL ์ธ์ฝ๋ฉ ์ ์ฉ)
|
| 11 |
-
base_url = "https://search.naver.com/search.naver?ssc=tab.news.all&where=news&sm=tab_jum&query="
|
| 12 |
-
encoded_keyword = urllib.parse.quote(keyword)
|
| 13 |
-
url = base_url + encoded_keyword
|
| 14 |
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
if response.status_code != 200:
|
| 19 |
-
|
| 20 |
-
|
|
|
|
| 21 |
# HTML ํ์ฑ
|
| 22 |
soup = BeautifulSoup(response.text, "html.parser")
|
| 23 |
-
|
| 24 |
-
# ๋ด์ค ๊ธฐ์ฌ ํญ๋ชฉ ์ถ์ถ (div ํ๊ทธ ์ค class๊ฐ "news_wrap api_ani_send" ์ธ ์์)
|
| 25 |
-
news_items = soup.select("div.news_wrap.api_ani_send")
|
| 26 |
-
news_list = []
|
| 27 |
-
|
| 28 |
-
# ๊ฐ ๋ด์ค ํญ๋ชฉ๋ณ๋ก ์ ๋ณด ์ถ์ถ
|
| 29 |
-
for item in news_items:
|
| 30 |
-
# 1. ์ ๋ฌธ์ฌ ์ ๋ณด: a ํ๊ทธ์ ํด๋์ค "info press"
|
| 31 |
-
press_tag = item.select_one("a.info.press")
|
| 32 |
-
newspaper = press_tag.get_text(strip=True) if press_tag else ""
|
| 33 |
-
|
| 34 |
-
# 2. ๋ฐํ์ผ ์ ๋ณด: div.info_group ๋ด์ span ํ๊ทธ ์ค '์ '์ด ํฌํจ๋ ์์ ํ์
|
| 35 |
-
publication = ""
|
| 36 |
-
for span in item.select("div.info_group span.info"):
|
| 37 |
-
if "์ " in span.get_text():
|
| 38 |
-
publication = span.get_text(strip=True)
|
| 39 |
-
break
|
| 40 |
-
|
| 41 |
-
# 3. ์ ๋ชฉ ๋ฐ ๋งํฌ: a ํ๊ทธ์ ํด๋์ค "news_tit"
|
| 42 |
-
title_tag = item.select_one("a.news_tit")
|
| 43 |
-
title = title_tag.get_text(strip=True) if title_tag else ""
|
| 44 |
-
link = title_tag["href"] if title_tag and "href" in title_tag.attrs else ""
|
| 45 |
-
|
| 46 |
-
# 4. ๋ด์ค ๊ฐ๋ต์ ๋ณด: div.news_dsc ๋ด์ dsc_wrap ์์ a ํ๊ทธ
|
| 47 |
-
summary_tag = item.select_one("div.news_dsc .dsc_wrap a")
|
| 48 |
-
summary = summary_tag.get_text(strip=True) if summary_tag else ""
|
| 49 |
-
|
| 50 |
-
news_list.append({
|
| 51 |
-
"์ ๋ฌธ์ฌ": newspaper,
|
| 52 |
-
"๋ฐํ์ผ": publication,
|
| 53 |
-
"์ ๋ชฉ": title,
|
| 54 |
-
"๋ด์ค๊ฐ๋ต์ ๋ณด": summary,
|
| 55 |
-
"๋งํฌ": link
|
| 56 |
-
})
|
| 57 |
-
|
| 58 |
-
# ๊ฒ์ ๊ฒฐ๊ณผ๊ฐ ์์ ๊ฒฝ์ฐ ์ฒ๋ฆฌ
|
| 59 |
-
if not news_list:
|
| 60 |
-
return "๊ฒ์ ๊ฒฐ๊ณผ๊ฐ ์์ต๋๋ค.", None
|
| 61 |
-
|
| 62 |
-
# DataFrame ์์ฑ
|
| 63 |
-
df = pd.DataFrame(news_list)
|
| 64 |
-
|
| 65 |
-
# UI์ฉ HTML ํ
์ด๋ธ (๋งํฌ๋ฅผ ํด๋ฆญ ๊ฐ๋ฅํ ํ์ดํผ๋งํฌ๋ก ๋ณํ)
|
| 66 |
-
df_html = df.copy()
|
| 67 |
-
df_html["๋งํฌ"] = df_html["๋งํฌ"].apply(lambda url: f'<a href="{url}" target="_blank">{url}</a>')
|
| 68 |
-
html_table = df_html.to_html(escape=False, index=False)
|
| 69 |
-
|
| 70 |
-
# ์์
๋ค์ด๋ก๋์ฉ DataFrame (๋งํฌ๋ฅผ ํ์ดํผ๋งํฌ ์์์ผ๋ก ๋ณํํ์ฌ ํด๋ฆญ ๊ฐ๋ฅํ๋๋ก ์ฒ๋ฆฌ)
|
| 71 |
-
df_excel = df.copy()
|
| 72 |
-
df_excel["๋งํฌ"] = df_excel["๋งํฌ"].apply(lambda url: f'=HYPERLINK("{url}", "{url}")')
|
| 73 |
-
|
| 74 |
-
# ์์ ์์
ํ์ผ ์์ฑ (ํ์ผ ๊ฒฝ๋ก ๋ฐํ)
|
| 75 |
-
tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".xlsx")
|
| 76 |
-
tmp_file.close() # ํ์ผ ๊ฒฝ๋ก๋ง ํ์ํ๋ฏ๋ก ๋ซ์
|
| 77 |
-
|
| 78 |
-
with pd.ExcelWriter(tmp_file.name, engine="openpyxl") as writer:
|
| 79 |
-
df_excel.to_excel(writer, index=False, sheet_name="Sheet1")
|
| 80 |
-
|
| 81 |
-
return html_table, tmp_file.name
|
| 82 |
-
|
| 83 |
-
# Gradio Blocks๋ฅผ ์ด์ฉํ ๋ ์ด์์ ๊ตฌ์ฑ (์ถ๋ ฅ ๊ฒฐ๊ณผ๋ฅผ ์์ง์ผ๋ก ๋ฐฐ์น)
|
| 84 |
-
with gr.Blocks() as demo:
|
| 85 |
-
gr.Markdown("# ๋ค์ด๋ฒ ๋ด์ค ์คํฌ๋ํ")
|
| 86 |
-
gr.Markdown("ํค์๋๋ฅผ ์
๋ ฅํ๋ฉด ๋ค์ด๋ฒ ๋ด์ค๋ฅผ ์คํฌ๋ํํ์ฌ ๊ฒฐ๊ณผ๋ฅผ ํ๋ก ๋ณด์ฌ์ฃผ๊ณ , ์์
ํ์ผ๋ก ๋ค์ด๋ก๋ํ ์ ์์ต๋๋ค.")
|
| 87 |
|
| 88 |
-
|
| 89 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
-
|
| 92 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
|
| 96 |
if __name__ == "__main__":
|
| 97 |
-
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
import requests
|
| 3 |
from bs4 import BeautifulSoup
|
| 4 |
+
import logging
|
| 5 |
+
import random
|
| 6 |
+
import time
|
|
|
|
| 7 |
|
| 8 |
+
# ๋ก๊น
์ค์ (๋๋ฒ๊น
์ ๋ณด๋ฅผ ์ฝ์์ ์ถ๋ ฅ)
|
| 9 |
+
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
+
def scrape_blog(url: str) -> str:
|
| 12 |
+
logging.info("์คํฌ๋ํ ์์: ์
๋ ฅ URL: %s", url)
|
| 13 |
+
|
| 14 |
+
# ์์ฒญ ์ ๋๋ค ๋๋ ์ด (1~3์ด)
|
| 15 |
+
delay = random.uniform(1, 3)
|
| 16 |
+
logging.debug("์์ฒญ ์ ๋๋ค ๋๋ ์ด: %.2f์ด", delay)
|
| 17 |
+
time.sleep(delay)
|
| 18 |
+
|
| 19 |
+
# ์ฌ๋ฌ ๋ธ๋ผ์ฐ์ ์ User-Agent ๋ฌธ์์ด ๋ฆฌ์คํธ
|
| 20 |
+
user_agents = [
|
| 21 |
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
|
| 22 |
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15",
|
| 23 |
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
|
| 24 |
+
"Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36"
|
| 25 |
+
]
|
| 26 |
+
chosen_user_agent = random.choice(user_agents)
|
| 27 |
+
logging.debug("์ ํ๋ User-Agent: %s", chosen_user_agent)
|
| 28 |
+
|
| 29 |
+
# ์์ฒญ ํค๋ ์ค์ (Referer๋ ๋ค์ด๋ฒ ๋ชจ๋ฐ์ผ ํ์ด์ง๋ก ์ค์ )
|
| 30 |
+
headers = {
|
| 31 |
+
"User-Agent": chosen_user_agent,
|
| 32 |
+
"Referer": "https://m.blog.naver.com/",
|
| 33 |
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
| 34 |
+
"Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
|
| 35 |
+
"Connection": "keep-alive"
|
| 36 |
+
}
|
| 37 |
+
logging.debug("์์ฒญ ํค๋: %s", headers)
|
| 38 |
+
|
| 39 |
+
try:
|
| 40 |
+
response = requests.get(url, headers=headers)
|
| 41 |
+
logging.info("์๋ต ์ํ ์ฝ๋: %s", response.status_code)
|
| 42 |
+
except Exception as e:
|
| 43 |
+
logging.error("์์ฒญ ์ค ์์ธ ๋ฐ์: %s", e)
|
| 44 |
+
return "์ค๋ฅ: URL ์์ฒญ ์ค ๋ฌธ์ ๊ฐ ๋ฐ์ํ์ต๋๋ค."
|
| 45 |
+
|
| 46 |
if response.status_code != 200:
|
| 47 |
+
logging.error("๋น์ ์ ์๋ต ์ํ ์ฝ๋: %s", response.status_code)
|
| 48 |
+
return f"์ค๋ฅ: ์์ฒญ ์คํจ (์ํ ์ฝ๋ {response.status_code})."
|
| 49 |
+
|
| 50 |
# HTML ํ์ฑ
|
| 51 |
soup = BeautifulSoup(response.text, "html.parser")
|
| 52 |
+
logging.info("HTML ํ์ฑ ์๋ฃ")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
+
# ์ ๋ชฉ ์ถ์ถ (์ ๊ณต๋ HTML ๊ตฌ์กฐ์ ๊ธฐ๋ฐํ์ฌ <span> ํ๊ทธ์ id๊ฐ 'SE-'๋ก ์์ํ๋ ์์ ์ ํ)
|
| 55 |
+
title_tag = soup.find("span", id=lambda x: x and x.startswith("SE-"))
|
| 56 |
+
if title_tag:
|
| 57 |
+
title = title_tag.get_text(strip=True)
|
| 58 |
+
logging.debug("์ ๋ชฉ ์ถ์ถ ์ฑ๊ณต: %s", title)
|
| 59 |
+
else:
|
| 60 |
+
logging.warning("์ ๋ชฉ ์ถ์ถ ์คํจ")
|
| 61 |
+
title = "์ ๋ชฉ์ ์ฐพ์ ์ ์์ต๋๋ค."
|
| 62 |
|
| 63 |
+
# ๋ณธ๋ฌธ ์ถ์ถ (๋ณธ๋ฌธ์ <div class="se-viewer se-theme-default" lang="ko-KR"> ๋ด๋ถ์ ์์น)
|
| 64 |
+
body_tag = soup.find("div", class_="se-viewer se-theme-default", lang="ko-KR")
|
| 65 |
+
if body_tag:
|
| 66 |
+
body = body_tag.get_text(separator="\n", strip=True)
|
| 67 |
+
logging.debug("๋ณธ๋ฌธ ์ถ์ถ ์ฑ๊ณต")
|
| 68 |
+
else:
|
| 69 |
+
logging.warning("๋ณธ๋ฌธ ์ถ์ถ ์คํจ")
|
| 70 |
+
body = "๋ณธ๋ฌธ ๋ด์ฉ์ ์ฐพ์ ์ ์์ต๋๋ค."
|
| 71 |
|
| 72 |
+
result = f"์ ๋ชฉ: {title}\n\n๋ณธ๋ฌธ:\n{body}"
|
| 73 |
+
logging.info("์คํฌ๋ํ ์๋ฃ")
|
| 74 |
+
return result
|
| 75 |
+
|
| 76 |
+
# Gradio ์ธํฐํ์ด์ค ์ค์
|
| 77 |
+
iface = gr.Interface(
|
| 78 |
+
fn=scrape_blog,
|
| 79 |
+
inputs=gr.Textbox(lines=1, placeholder="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ URL์ ์
๋ ฅํ์ธ์."),
|
| 80 |
+
outputs=gr.Textbox(label="์คํฌ๋ํ ๊ฒฐ๊ณผ", lines=20),
|
| 81 |
+
title="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ์คํฌ๋ํ",
|
| 82 |
+
description="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ๋งํฌ๋ฅผ ์
๋ ฅํ๋ฉด ์ ๋ชฉ๊ณผ ๋ณธ๋ฌธ์ ์ถ์ถํฉ๋๋ค. (๋๋ฒ๊น
์ ๋ณด๋ ์ฝ์์ ์ถ๋ ฅ๋ฉ๋๋ค.)"
|
| 83 |
+
)
|
| 84 |
|
| 85 |
if __name__ == "__main__":
|
| 86 |
+
iface.launch()
|