Sangwon78 commited on
Commit
76eecf3
ยท
verified ยท
1 Parent(s): cc4b5d8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +57 -77
app.py CHANGED
@@ -1,107 +1,87 @@
 
1
  import requests
2
- import random
3
  import time
 
4
  import logging
5
- from bs4 import BeautifulSoup
6
- import gradio as gr
7
 
8
- # ๋””๋ฒ„๊น… ๋กœ๊ทธ ์„ค์ • (UI์— ๋…ธ์ถœ๋˜์ง€ ์•Š๊ณ  debug.log ํŒŒ์ผ์— ๊ธฐ๋ก๋จ)
9
- logging.basicConfig(
10
- filename='debug.log',
11
- level=logging.DEBUG,
12
- format='%(asctime)s - %(levelname)s - %(message)s'
13
- )
14
- logging.debug("์• ํ”Œ๋ฆฌ์ผ€์ด์…˜ ์‹œ์ž‘.")
15
 
16
- # ์—ฌ๋Ÿฌ ๋ธŒ๋ผ์šฐ์ €์˜ User-Agent ๋ฌธ์ž์—ด ๋ฆฌ์ŠคํŠธ
17
  USER_AGENTS = [
18
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
19
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0",
20
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
21
- # ํ•„์š”์‹œ ์ถ”๊ฐ€ User-Agent ๋ฌธ์ž์—ด์„ ๋„ฃ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค.
22
  ]
23
 
24
- def get_random_user_agent():
25
- user_agent = random.choice(USER_AGENTS)
26
- logging.debug(f"์„ ํƒ๋œ User-Agent: {user_agent}")
27
- return user_agent
28
 
29
- def scrape_naver_blog(url):
30
- logging.debug("์Šคํฌ๋ž˜ํ•‘ ํ”„๋กœ์„ธ์Šค ์‹œ์ž‘.")
31
- # ์š”์ฒญ ์ „ ๋žœ๋ค ๋”œ๋ ˆ์ด (1~3์ดˆ)
32
  delay = random.uniform(1, 3)
33
- logging.debug(f"์š”์ฒญ ์ „ {delay:.2f}์ดˆ ๋Œ€๊ธฐ.")
34
  time.sleep(delay)
35
-
36
- # HTTP ํ—ค๋” ์„ค์ • (์‚ฌ๋žŒ์ฒ˜๋Ÿผ ๋ณด์ด๋„๋ก User-Agent, Referer ๋“ฑ ์„ค์ •)
 
37
  headers = {
38
- "User-Agent": get_random_user_agent(),
39
- "Referer": "https://m.blog.naver.com",
40
  "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7"
41
  }
42
- logging.debug(f"์š”์ฒญ ํ—ค๋”: {headers}")
43
-
44
  try:
45
- logging.debug(f"URL๋กœ GET ์š”์ฒญ ์ „์†ก: {url}")
46
  response = requests.get(url, headers=headers, timeout=10)
47
- logging.debug(f"์‘๋‹ต ์ƒํƒœ ์ฝ”๋“œ: {response.status_code}")
 
 
 
48
  except Exception as e:
49
- logging.error(f"HTTP ์š”์ฒญ ์ค‘ ์—๋Ÿฌ ๋ฐœ์ƒ: {e}")
50
- return f"HTTP ์š”์ฒญ ์ค‘ ์—๋Ÿฌ ๋ฐœ์ƒ: {e}"
51
-
52
- if response.status_code != 200:
53
- logging.error("200์ด ์•„๋‹Œ ์ƒํƒœ ์ฝ”๋“œ ์ˆ˜์‹ .")
54
- return f"HTTP ์š”์ฒญ ์‹คํŒจ, ์ƒํƒœ ์ฝ”๋“œ: {response.status_code}"
55
-
56
  # HTML ํŒŒ์‹ฑ
57
- logging.debug("HTML ํŒŒ์‹ฑ ์‹œ์ž‘.")
58
  soup = BeautifulSoup(response.text, "html.parser")
59
-
60
- # ์ œ๋ชฉ ์ถ”์ถœ
61
- logging.debug("์ œ๋ชฉ ์ถ”์ถœ ์‹œ๋„.")
62
  title_div = soup.find("div", class_="se-module se-module-text se-title-text")
63
  if title_div:
64
- title_p = title_div.find("p")
65
- if title_p:
66
- title_span = title_p.find("span")
67
- if title_span:
68
- title_text = title_span.get_text(strip=True)
69
- logging.debug(f"์ถ”์ถœ๋œ ์ œ๋ชฉ: {title_text}")
70
- else:
71
- title_text = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
72
- logging.error("์ œ๋ชฉ์„ ํฌํ•จํ•˜๋Š” span ํƒœ๊ทธ๋ฅผ ์ฐพ์„ ์ˆ˜ ์—†์Œ.")
73
  else:
74
- title_text = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
75
- logging.error("์ œ๋ชฉ์„ ํฌํ•จํ•˜๋Š” p ํƒœ๊ทธ๋ฅผ ์ฐพ์„ ์ˆ˜ ์—†์Œ.")
76
  else:
77
- title_text = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
78
- logging.error("์ œ๋ชฉ์„ ํฌํ•จํ•˜๋Š” div ํƒœ๊ทธ๋ฅผ ์ฐพ์„ ์ˆ˜ ์—†์Œ.")
79
-
80
- # ๋ณธ๋ฌธ ์ถ”์ถœ
81
- logging.debug("๋ณธ๋ฌธ ๋‚ด์šฉ ์ถ”์ถœ ์‹œ๋„.")
82
- content_div = soup.find("div", class_="se-main-container")
83
- if content_div:
84
- content_text = content_div.get_text(separator="\n", strip=True)
85
- logging.debug("๋ณธ๋ฌธ ๋‚ด์šฉ ์ถ”์ถœ ์„ฑ๊ณต.")
86
  else:
87
- content_text = "๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
88
- logging.error("๋ณธ๋ฌธ ๋‚ด์šฉ์„ ํฌํ•จํ•˜๋Š” div ํƒœ๊ทธ๋ฅผ ์ฐพ์„ ์ˆ˜ ์—†์Œ.")
89
-
90
- # ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ํ•˜๋‚˜์˜ ํ…์ŠคํŠธ๋กœ ๊ฒฐํ•ฉ
91
- combined_text = f"์ œ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}"
92
- logging.debug("์Šคํฌ๋ž˜ํ•‘ ํ”„๋กœ์„ธ์Šค ์™„๋ฃŒ.")
93
- return combined_text
94
 
95
- # Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ (UI์— ๋””๋ฒ„๊ทธ ๋กœ๊ทธ๋Š” ๋…ธ์ถœ๋˜์ง€ ์•Š์Œ)
96
- logging.debug("Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ.")
97
  iface = gr.Interface(
98
- fn=scrape_naver_blog,
99
- inputs=gr.Textbox(label="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ", placeholder="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ URL์„ ์ž…๋ ฅํ•˜์„ธ์š”"),
100
- outputs=gr.Textbox(label="์Šคํฌ๋ž˜ํ•‘ ๊ฒฐ๊ณผ", lines=20),
101
- title="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ์Šคํฌ๋ž˜ํ•‘ ๋„๊ตฌ",
102
- description="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค."
103
  )
104
 
105
  if __name__ == "__main__":
106
- logging.debug("Gradio ์•ฑ ์‹คํ–‰.")
107
- iface.launch()
 
1
+ import gradio as gr
2
  import requests
3
+ from bs4 import BeautifulSoup
4
  import time
5
+ import random
6
  import logging
 
 
7
 
8
+ # ๋””๋ฒ„๊น… ๋กœ๊ทธ ์„ค์ • (UI์— ์ถœ๋ ฅ๋˜์ง€ ์•Š๊ณ  ์ฝ˜์†”์— ์ถœ๋ ฅ)
9
+ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
 
 
 
 
10
 
11
+ # ๋‹ค์–‘ํ•œ ๋ธŒ๋ผ์šฐ์ €์˜ User-Agent ๋ฌธ์ž์—ด ๋ฆฌ์ŠคํŠธ
12
  USER_AGENTS = [
13
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
14
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
15
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",
16
+ "Mozilla/5.0 (Linux; Android 10; SM-G973N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36"
17
  ]
18
 
19
+ def scrape_blog(url):
20
+ logging.debug("์Šคํฌ๋ž˜ํ•‘ ์‹œ์ž‘: %s", url)
 
 
21
 
22
+ # ์š”์ฒญ ์ „์— 1~3์ดˆ ๋žœ๋ค ๋”œ๋ ˆ์ด ์ ์šฉ (์„œ๋ฒ„ ๋ถ€ํ•˜ ๋ฐ ๋ด‡ ๊ฐ์ง€ ํšŒํ”ผ)
 
 
23
  delay = random.uniform(1, 3)
24
+ logging.debug("์š”์ฒญ ์ „ ๋žœ๋ค ๋”œ๋ ˆ์ด: %.2f์ดˆ", delay)
25
  time.sleep(delay)
26
+
27
+ # ๋žœ๋ค User-Agent ์„ ํƒ ๋ฐ Referer, Accept-Language ์„ค์ • (์›น์ฐจ๋‹จ๋ฐฉ์ง€ ๊ธฐ๋Šฅ)
28
+ user_agent = random.choice(USER_AGENTS)
29
  headers = {
30
+ "User-Agent": user_agent,
31
+ "Referer": "https://m.blog.naver.com", # ๋„ค์ด๋ฒ„ ๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€์—์„œ ์˜จ ๊ฒƒ์ฒ˜๋Ÿผ ์„ค์ •
32
  "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7"
33
  }
34
+ logging.debug("์„ค์ •๋œ HTTP ํ—ค๋”: %s", headers)
35
+
36
  try:
 
37
  response = requests.get(url, headers=headers, timeout=10)
38
+ logging.debug("HTTP ์š”์ฒญ ์™„๋ฃŒ - ์ƒํƒœ ์ฝ”๋“œ: %s", response.status_code)
39
+ if response.status_code != 200:
40
+ logging.error("HTTP ์š”์ฒญ ์‹คํŒจ: ์ƒํƒœ ์ฝ”๋“œ %s", response.status_code)
41
+ return f"HTTP ์š”์ฒญ ์‹คํŒจ: ์ƒํƒœ ์ฝ”๋“œ {response.status_code}"
42
  except Exception as e:
43
+ logging.exception("HTTP ์š”์ฒญ ์ค‘ ์˜ˆ์™ธ ๋ฐœ์ƒ")
44
+ return f"HTTP ์š”์ฒญ ์ค‘ ์˜ˆ์™ธ ๋ฐœ์ƒ: {e}"
45
+
 
 
 
 
46
  # HTML ํŒŒ์‹ฑ
 
47
  soup = BeautifulSoup(response.text, "html.parser")
48
+ logging.debug("HTML ํŒŒ์‹ฑ ์™„๋ฃŒ")
49
+
50
+ # ์ œ๋ชฉ ์ถ”์ถœ (HTML ๊ตฌ์กฐ ์˜ˆ์‹œ ์ฐธ๊ณ )
51
  title_div = soup.find("div", class_="se-module se-module-text se-title-text")
52
  if title_div:
53
+ title_tag = title_div.find(["p", "span"])
54
+ if title_tag:
55
+ title_text = title_tag.get_text(strip=True)
 
 
 
 
 
 
56
  else:
57
+ title_text = ""
 
58
  else:
59
+ title_text = ""
60
+ logging.debug("์ถ”์ถœ๋œ ์ œ๋ชฉ: %s", title_text)
61
+
62
+ # ๋ณธ๋ฌธ ์ถ”์ถœ (HTML ๊ตฌ์กฐ ๋‚ด 'se-main-container' ํด๋ž˜์Šค ์‚ฌ์šฉ)
63
+ body_div = soup.find("div", class_="se-main-container")
64
+ if body_div:
65
+ # ๋ณธ๋ฌธ ๋‚ด ๋ชจ๋“  ํ…์ŠคํŠธ๋ฅผ ์ถ”์ถœํ•˜๋ฉฐ ๊ฐ ์š”์†Œ ์‚ฌ์ด์— ์ค„๋ฐ”๊ฟˆ ์ ์šฉ
66
+ body_text = body_div.get_text(separator="\n", strip=True)
 
67
  else:
68
+ body_text = ""
69
+ logging.debug("์ถ”์ถœ๋œ ๋ณธ๋ฌธ ๊ธธ์ด: %d", len(body_text))
70
+
71
+ # ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ์„ ํ•˜๋‚˜์˜ ํ…์ŠคํŠธ๋กœ ๊ฒฐํ•ฉํ•˜์—ฌ ๋ฐ˜ํ™˜
72
+ result = f"์ œ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{body_text}"
73
+ logging.debug("์ตœ์ข… ๊ฒฐ๊ณผ ์ƒ์„ฑ ์™„๋ฃŒ")
74
+ return result
75
 
76
+ # Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ
 
77
  iface = gr.Interface(
78
+ fn=scrape_blog,
79
+ inputs=gr.inputs.Textbox(label="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜์„ธ์š”."),
80
+ outputs=gr.outputs.Textbox(label="์Šคํฌ๋ž˜ํ•‘ ๊ฒฐ๊ณผ"),
81
+ title="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ์Šคํฌ๋ž˜ํ•‘",
82
+ description="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์Šคํฌ๋ž˜ํ•‘ํ•˜์—ฌ ์ถœ๋ ฅํ•ฉ๋‹ˆ๋‹ค."
83
  )
84
 
85
  if __name__ == "__main__":
86
+ logging.debug("Gradio ์ธํ„ฐํŽ˜์ด์Šค ์‹คํ–‰ ์ค€๋น„")
87
+ iface.launch()