Sangwon78 commited on
Commit
49d3cf8
ยท
verified ยท
1 Parent(s): 147e3b6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +74 -85
app.py CHANGED
@@ -1,97 +1,86 @@
 
1
  import requests
2
  from bs4 import BeautifulSoup
3
- import gradio as gr
4
- import pandas as pd
5
- import urllib.parse
6
- import tempfile
7
 
8
- # ๋„ค์ด๋ฒ„ ๋‰ด์Šค ์Šคํฌ๋ž˜ํ•‘ ํ•จ์ˆ˜
9
- def scrape_naver_news(keyword):
10
- # ํ‚ค์›Œ๋“œ ๊ธฐ๋ฐ˜ ๋„ค์ด๋ฒ„ ๋‰ด์Šค ๊ฒ€์ƒ‰ URL ์ƒ์„ฑ (URL ์ธ์ฝ”๋”ฉ ์ ์šฉ)
11
- base_url = "https://search.naver.com/search.naver?ssc=tab.news.all&where=news&sm=tab_jum&query="
12
- encoded_keyword = urllib.parse.quote(keyword)
13
- url = base_url + encoded_keyword
14
 
15
- # HTTP ์š”์ฒญ (User-Agent ํ—ค๋” ์ถ”๊ฐ€)
16
- headers = {"User-Agent": "Mozilla/5.0"}
17
- response = requests.get(url, headers=headers)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  if response.status_code != 200:
19
- return "๋‰ด์Šค ์ •๋ณด๋ฅผ ๊ฐ€์ ธ์˜ค์ง€ ๋ชปํ–ˆ์Šต๋‹ˆ๋‹ค.", None
20
-
 
21
  # HTML ํŒŒ์‹ฑ
22
  soup = BeautifulSoup(response.text, "html.parser")
23
-
24
- # ๋‰ด์Šค ๊ธฐ์‚ฌ ํ•ญ๋ชฉ ์ถ”์ถœ (div ํƒœ๊ทธ ์ค‘ class๊ฐ€ "news_wrap api_ani_send" ์ธ ์š”์†Œ)
25
- news_items = soup.select("div.news_wrap.api_ani_send")
26
- news_list = []
27
-
28
- # ๊ฐ ๋‰ด์Šค ํ•ญ๋ชฉ๋ณ„๋กœ ์ •๋ณด ์ถ”์ถœ
29
- for item in news_items:
30
- # 1. ์‹ ๋ฌธ์‚ฌ ์ •๋ณด: a ํƒœ๊ทธ์˜ ํด๋ž˜์Šค "info press"
31
- press_tag = item.select_one("a.info.press")
32
- newspaper = press_tag.get_text(strip=True) if press_tag else ""
33
-
34
- # 2. ๋ฐœํ–‰์ผ ์ •๋ณด: div.info_group ๋‚ด์˜ span ํƒœ๊ทธ ์ค‘ '์ „'์ด ํฌํ•จ๋œ ์š”์†Œ ํƒ์ƒ‰
35
- publication = ""
36
- for span in item.select("div.info_group span.info"):
37
- if "์ „" in span.get_text():
38
- publication = span.get_text(strip=True)
39
- break
40
-
41
- # 3. ์ œ๋ชฉ ๋ฐ ๋งํฌ: a ํƒœ๊ทธ์˜ ํด๋ž˜์Šค "news_tit"
42
- title_tag = item.select_one("a.news_tit")
43
- title = title_tag.get_text(strip=True) if title_tag else ""
44
- link = title_tag["href"] if title_tag and "href" in title_tag.attrs else ""
45
-
46
- # 4. ๋‰ด์Šค ๊ฐ„๋žต์ •๋ณด: div.news_dsc ๋‚ด์˜ dsc_wrap ์•ˆ์˜ a ํƒœ๊ทธ
47
- summary_tag = item.select_one("div.news_dsc .dsc_wrap a")
48
- summary = summary_tag.get_text(strip=True) if summary_tag else ""
49
-
50
- news_list.append({
51
- "์‹ ๋ฌธ์‚ฌ": newspaper,
52
- "๋ฐœํ–‰์ผ": publication,
53
- "์ œ๋ชฉ": title,
54
- "๋‰ด์Šค๊ฐ„๋žต์ •๋ณด": summary,
55
- "๋งํฌ": link
56
- })
57
-
58
- # ๊ฒ€์ƒ‰ ๊ฒฐ๊ณผ๊ฐ€ ์—†์„ ๊ฒฝ์šฐ ์ฒ˜๋ฆฌ
59
- if not news_list:
60
- return "๊ฒ€์ƒ‰ ๊ฒฐ๊ณผ๊ฐ€ ์—†์Šต๋‹ˆ๋‹ค.", None
61
-
62
- # DataFrame ์ƒ์„ฑ
63
- df = pd.DataFrame(news_list)
64
-
65
- # UI์šฉ HTML ํ…Œ์ด๋ธ” (๋งํฌ๋ฅผ ํด๋ฆญ ๊ฐ€๋Šฅํ•œ ํ•˜์ดํผ๋งํฌ๋กœ ๋ณ€ํ™˜)
66
- df_html = df.copy()
67
- df_html["๋งํฌ"] = df_html["๋งํฌ"].apply(lambda url: f'<a href="{url}" target="_blank">{url}</a>')
68
- html_table = df_html.to_html(escape=False, index=False)
69
-
70
- # ์—‘์…€ ๋‹ค์šด๋กœ๋“œ์šฉ DataFrame (๋งํฌ๋ฅผ ํ•˜์ดํผ๋งํฌ ์ˆ˜์‹์œผ๋กœ ๋ณ€ํ™˜ํ•˜์—ฌ ํด๋ฆญ ๊ฐ€๋Šฅํ•˜๋„๋ก ์ฒ˜๋ฆฌ)
71
- df_excel = df.copy()
72
- df_excel["๋งํฌ"] = df_excel["๋งํฌ"].apply(lambda url: f'=HYPERLINK("{url}", "{url}")')
73
-
74
- # ์ž„์‹œ ์—‘์…€ ํŒŒ์ผ ์ƒ์„ฑ (ํŒŒ์ผ ๊ฒฝ๋กœ ๋ฐ˜ํ™˜)
75
- tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".xlsx")
76
- tmp_file.close() # ํŒŒ์ผ ๊ฒฝ๋กœ๋งŒ ํ•„์š”ํ•˜๋ฏ€๋กœ ๋‹ซ์Œ
77
-
78
- with pd.ExcelWriter(tmp_file.name, engine="openpyxl") as writer:
79
- df_excel.to_excel(writer, index=False, sheet_name="Sheet1")
80
-
81
- return html_table, tmp_file.name
82
-
83
- # Gradio Blocks๋ฅผ ์ด์šฉํ•œ ๋ ˆ์ด์•„์›ƒ ๊ตฌ์„ฑ (์ถœ๋ ฅ ๊ฒฐ๊ณผ๋ฅผ ์ˆ˜์ง์œผ๋กœ ๋ฐฐ์น˜)
84
- with gr.Blocks() as demo:
85
- gr.Markdown("# ๋„ค์ด๋ฒ„ ๋‰ด์Šค ์Šคํฌ๋ž˜ํ•‘")
86
- gr.Markdown("ํ‚ค์›Œ๋“œ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด ๋„ค์ด๋ฒ„ ๋‰ด์Šค๋ฅผ ์Šคํฌ๋ž˜ํ•‘ํ•˜์—ฌ ๊ฒฐ๊ณผ๋ฅผ ํ‘œ๋กœ ๋ณด์—ฌ์ฃผ๊ณ , ์—‘์…€ ํŒŒ์ผ๋กœ ๋‹ค์šด๋กœ๋“œํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค.")
87
 
88
- keyword = gr.Textbox(label="ํ‚ค์›Œ๋“œ๋ฅผ ์ž…๋ ฅํ•˜์„ธ์š”")
89
- btn = gr.Button("๊ฒ€์ƒ‰")
 
 
 
 
 
 
90
 
91
- html_output = gr.HTML(label="๋‰ด์Šค ๊ฒฐ๊ณผ")
92
- file_output = gr.File(label="์—‘์…€ ๋‹ค์šด๋กœ๋“œ")
 
 
 
 
 
 
93
 
94
- btn.click(fn=scrape_naver_news, inputs=keyword, outputs=[html_output, file_output])
 
 
 
 
 
 
 
 
 
 
 
95
 
96
  if __name__ == "__main__":
97
- demo.launch()
 
1
+ import gradio as gr
2
  import requests
3
  from bs4 import BeautifulSoup
4
+ import logging
5
+ import random
6
+ import time
 
7
 
8
+ # ๋กœ๊น… ์„ค์ • (๋””๋ฒ„๊น… ์ •๋ณด๋ฅผ ์ฝ˜์†”์— ์ถœ๋ ฅ)
9
+ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
 
 
 
10
 
11
+ def scrape_blog(url: str) -> str:
12
+ logging.info("์Šคํฌ๋ž˜ํ•‘ ์‹œ์ž‘: ์ž…๋ ฅ URL: %s", url)
13
+
14
+ # ์š”์ฒญ ์ „ ๋žœ๋ค ๋”œ๋ ˆ์ด (1~3์ดˆ)
15
+ delay = random.uniform(1, 3)
16
+ logging.debug("์š”์ฒญ ์ „ ๋žœ๋ค ๋”œ๋ ˆ์ด: %.2f์ดˆ", delay)
17
+ time.sleep(delay)
18
+
19
+ # ์—ฌ๋Ÿฌ ๋ธŒ๋ผ์šฐ์ €์˜ User-Agent ๋ฌธ์ž์—ด ๋ฆฌ์ŠคํŠธ
20
+ user_agents = [
21
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
22
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15",
23
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
24
+ "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36"
25
+ ]
26
+ chosen_user_agent = random.choice(user_agents)
27
+ logging.debug("์„ ํƒ๋œ User-Agent: %s", chosen_user_agent)
28
+
29
+ # ์š”์ฒญ ํ—ค๋” ์„ค์ • (Referer๋Š” ๋„ค์ด๋ฒ„ ๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€๋กœ ์„ค์ •)
30
+ headers = {
31
+ "User-Agent": chosen_user_agent,
32
+ "Referer": "https://m.blog.naver.com/",
33
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
34
+ "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
35
+ "Connection": "keep-alive"
36
+ }
37
+ logging.debug("์š”์ฒญ ํ—ค๋”: %s", headers)
38
+
39
+ try:
40
+ response = requests.get(url, headers=headers)
41
+ logging.info("์‘๋‹ต ์ƒํƒœ ์ฝ”๋“œ: %s", response.status_code)
42
+ except Exception as e:
43
+ logging.error("์š”์ฒญ ์ค‘ ์˜ˆ์™ธ ๋ฐœ์ƒ: %s", e)
44
+ return "์˜ค๋ฅ˜: URL ์š”์ฒญ ์ค‘ ๋ฌธ์ œ๊ฐ€ ๋ฐœ์ƒํ–ˆ์Šต๋‹ˆ๋‹ค."
45
+
46
  if response.status_code != 200:
47
+ logging.error("๋น„์ •์ƒ ์‘๋‹ต ์ƒํƒœ ์ฝ”๋“œ: %s", response.status_code)
48
+ return f"์˜ค๋ฅ˜: ์š”์ฒญ ์‹คํŒจ (์ƒํƒœ ์ฝ”๋“œ {response.status_code})."
49
+
50
  # HTML ํŒŒ์‹ฑ
51
  soup = BeautifulSoup(response.text, "html.parser")
52
+ logging.info("HTML ํŒŒ์‹ฑ ์™„๋ฃŒ")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
 
54
+ # ์ œ๋ชฉ ์ถ”์ถœ (์ œ๊ณต๋œ HTML ๊ตฌ์กฐ์— ๊ธฐ๋ฐ˜ํ•˜์—ฌ <span> ํƒœ๊ทธ์˜ id๊ฐ€ 'SE-'๋กœ ์‹œ์ž‘ํ•˜๋Š” ์š”์†Œ ์„ ํƒ)
55
+ title_tag = soup.find("span", id=lambda x: x and x.startswith("SE-"))
56
+ if title_tag:
57
+ title = title_tag.get_text(strip=True)
58
+ logging.debug("์ œ๋ชฉ ์ถ”์ถœ ์„ฑ๊ณต: %s", title)
59
+ else:
60
+ logging.warning("์ œ๋ชฉ ์ถ”์ถœ ์‹คํŒจ")
61
+ title = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
62
 
63
+ # ๋ณธ๋ฌธ ์ถ”์ถœ (๋ณธ๋ฌธ์€ <div class="se-viewer se-theme-default" lang="ko-KR"> ๋‚ด๋ถ€์— ์œ„์น˜)
64
+ body_tag = soup.find("div", class_="se-viewer se-theme-default", lang="ko-KR")
65
+ if body_tag:
66
+ body = body_tag.get_text(separator="\n", strip=True)
67
+ logging.debug("๋ณธ๋ฌธ ์ถ”์ถœ ์„ฑ๊ณต")
68
+ else:
69
+ logging.warning("๋ณธ๋ฌธ ์ถ”์ถœ ์‹คํŒจ")
70
+ body = "๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
71
 
72
+ result = f"์ œ๋ชฉ: {title}\n\n๋ณธ๋ฌธ:\n{body}"
73
+ logging.info("์Šคํฌ๋ž˜ํ•‘ ์™„๋ฃŒ")
74
+ return result
75
+
76
+ # Gradio ์ธํ„ฐํŽ˜์ด์Šค ์„ค์ •
77
+ iface = gr.Interface(
78
+ fn=scrape_blog,
79
+ inputs=gr.Textbox(lines=1, placeholder="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ URL์„ ์ž…๋ ฅํ•˜์„ธ์š”."),
80
+ outputs=gr.Textbox(label="์Šคํฌ๋ž˜ํ•‘ ๊ฒฐ๊ณผ", lines=20),
81
+ title="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ์Šคํฌ๋ž˜ํ•‘",
82
+ description="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ์„ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค. (๋””๋ฒ„๊น… ์ •๋ณด๋Š” ์ฝ˜์†”์— ์ถœ๋ ฅ๋ฉ๋‹ˆ๋‹ค.)"
83
+ )
84
 
85
  if __name__ == "__main__":
86
+ iface.launch()