test1 / app.py
Sangwon78's picture
Update app.py
6de0076 verified
Raw History Blame Contribute Delete
7.3 kB
import requests
import random
import time
import logging
from bs4 import BeautifulSoup
import gradio as gr
from urllib.parse import urlparse, urlunparse
"""
[๊ธฐ๋ณธ์‚ฌํ•ญ]
# ํ•œ๊ตญ์–ด๋กœ ์„ค๋ช…
# ํ—ˆ๊น…ํŽ˜์ด์Šค์˜ ๊ทธ๋ผ๋””์˜ค ํ™˜๊ฒฝ์—์„œ ๋™์ž‘๋˜๋Š” ์ŠคํŽ˜์ด์Šค
# ์š”์ฒญํ•œ ์‚ฌํ•ญ์˜ ์ฝ”๋“œ๋ฅผ ์ž„์˜์ ์œผ๋กœ ์ถ•์†Œ/์‚ญ์ œํ•˜์ง€ ๋ง๊ณ  ์ „์ฒด์ฝ”๋“œ๋ฅผ ์ถœ๋ ฅ
[์š”์ฒญ์‚ฌํ•ญ]
1) ๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด, ํ•ด๋‹น ๋งํฌ์˜ ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์Šคํฌ๋ž˜ํ•‘
2) ์ถœ๋ ฅ์€ ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ๋‚ด์šฉ์ด ํ…์ŠคํŠธ์ฐฝ์— ํ•œ ๋ฒˆ์— ๋‚˜์˜ค๋„๋ก
3) ํƒ€๊ฒŸ URL ์˜ˆ์‹œ:
- ๊ธฐ๋ณธ: https://blog.naver.com/acoloje/223783254325
4) ๋ฐ์ดํ„ฐ HTML ๊ตฌ์กฐ(์ œ๋ชฉ):
<div class="se-module se-module-text se-title-text">
<p class="se-text-paragraph se-text-paragraph-align-" style="" id="SE-d4c7e111-1d97-40cf-a490-eb188c4b7f04">
<span style="" class="se-fs- se-ff-" id="SE-b62ca798-fd52-11ef-8346-2d7ed694d63c"><!-- -->์†๋ฐ”๋‹ฅ์— M ๋งˆํฌ๊ฐ€ ์žˆ์–ด์š”?<!-- --></span>
</p>
</div>
5) ๋ฐ์ดํ„ฐHTML ๊ตฌ์กฐ(๋ณธ๋ฌธ):
<div class="se-main-container">
...
<div class="se-component se-text se-l-default" ...>
...
<p class="se-text-paragraph ...">๋ณธ๋ฌธ ๋‚ด์šฉ</p>
...
</div>
...
</div>
[์ฐจ๋‹จ๋ฐฉ์ง€ ์ ์šฉ์ž๋ฃŒ]
1. Header ์„ค์ •ํ•˜๊ธฐ (User-Agent, Referer ๋“ฑ)
2. User-Agent ๋žœ๋ค ์„ค์ •
3. Referer ์„ค์ • (๋„ค์ด๋ฒ„ ๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€์—์„œ ์˜จ ๊ฒƒ์ฒ˜๋Ÿผ ๋ณด์ด๋„๋ก)
4. ๋žœ๋ค ๋”œ๋ ˆ์ด (1~3์ดˆ)
"""
# ๋””๋ฒ„๊น… ๋กœ๊ทธ ์„ค์ • (UI์—๋Š” ๋ณด์ด์ง€ ์•Š๊ณ  debug.log ํŒŒ์ผ์— ๊ธฐ๋ก)
logging.basicConfig(
filename='debug.log',
level=logging.DEBUG,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logging.debug("์• ํ”Œ๋ฆฌ์ผ€์ด์…˜ ์‹œ์ž‘.")
# ์—ฌ๋Ÿฌ ๋ธŒ๋ผ์šฐ์ €์˜ User-Agent ๋ฌธ์ž์—ด ๋ฆฌ์ŠคํŠธ
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
]
def get_random_user_agent():
user_agent = random.choice(USER_AGENTS)
logging.debug(f"์„ ํƒ๋œ User-Agent: {user_agent}")
return user_agent
def convert_to_mobile_url(url):
"""
PC ๋ฒ„์ „ blog.naver.com ์ฃผ์†Œ๋ฅผ m.blog.naver.com์œผ๋กœ ๋ณ€ํ™˜ํ•˜์—ฌ
๋ชจ๋ฐ”์ผ ํŽ˜์ด์ง€๋ฅผ ์š”์ฒญํ•  ์ˆ˜ ์žˆ๋„๋ก ํ•จ.
"""
parsed = urlparse(url)
if "blog.naver.com" in parsed.netloc and "m.blog.naver.com" not in parsed.netloc:
new_netloc = parsed.netloc.replace("blog.naver.com", "m.blog.naver.com")
new_url = urlunparse((parsed.scheme, new_netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
logging.debug(f"๋ชจ๋ฐ”์ผ ์ฃผ์†Œ๋กœ ๋ณ€ํ™˜: {new_url}")
return new_url
return url
def scrape_naver_blog(url):
logging.debug("์Šคํฌ๋ž˜ํ•‘ ํ”„๋กœ์„ธ์Šค ์‹œ์ž‘.")
# 1) URL์„ ๋ชจ๋ฐ”์ผ ๋ฒ„์ „์œผ๋กœ ๋ณ€ํ™˜
mobile_url = convert_to_mobile_url(url)
logging.debug(f"์ตœ์ข… ์š”์ฒญํ•  URL: {mobile_url}")
# 2) ์š”์ฒญ ์ „ ๋žœ๋ค ๋”œ๋ ˆ์ด (1~3์ดˆ)
delay = random.uniform(1, 3)
logging.debug(f"์š”์ฒญ ์ „ {delay:.2f}์ดˆ ๋Œ€๊ธฐ.")
time.sleep(delay)
# 3) HTTP ํ—ค๋” ์„ค์ • (User-Agent, Referer ๋“ฑ)
headers = {
"User-Agent": get_random_user_agent(),
"Referer": "https://m.blog.naver.com",
"Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7"
}
logging.debug(f"์š”์ฒญ ํ—ค๋”: {headers}")
# 4) GET ์š”์ฒญ
try:
logging.debug(f"URL๋กœ GET ์š”์ฒญ ์ „์†ก: {mobile_url}")
response = requests.get(mobile_url, headers=headers, timeout=10)
logging.debug(f"์‘๋‹ต ์ƒํƒœ ์ฝ”๋“œ: {response.status_code}")
except Exception as e:
logging.error(f"HTTP ์š”์ฒญ ์ค‘ ์—๋Ÿฌ ๋ฐœ์ƒ: {e}")
return f"HTTP ์š”์ฒญ ์ค‘ ์—๋Ÿฌ ๋ฐœ์ƒ: {e}"
if response.status_code != 200:
logging.error("200์ด ์•„๋‹Œ ์ƒํƒœ ์ฝ”๋“œ ์ˆ˜์‹ .")
return f"HTTP ์š”์ฒญ ์‹คํŒจ, ์ƒํƒœ ์ฝ”๋“œ: {response.status_code}"
# 5) HTML ํŒŒ์‹ฑ
logging.debug("HTML ํŒŒ์‹ฑ ์‹œ์ž‘.")
soup = BeautifulSoup(response.text, "html.parser")
# --------------------
# ์ œ๋ชฉ ์ถ”์ถœ ๋กœ์ง
# --------------------
logging.debug("์ œ๋ชฉ ์ถ”์ถœ ์‹œ๋„.")
# (A) ์ƒˆ ์—๋””ํ„ฐ ๊ตฌ์กฐ
title_text = None
title_div = soup.find("div", class_="se-module se-module-text se-title-text")
if title_div:
p_tag = title_div.find("p")
if p_tag:
span_tag = p_tag.find("span")
if span_tag:
title_text = span_tag.get_text(strip=True)
# (B) ๊ตฌ ์—๋””ํ„ฐ ๊ตฌ์กฐ: #postViewArea ๋‚ด๋ถ€ ๋“ฑ
if not title_text:
post_view_area = soup.find("div", id="postViewArea")
if post_view_area:
candidate_title = post_view_area.find("span", class_="pcol1 itemSubjectBoldfont")
if candidate_title:
title_text = candidate_title.get_text(strip=True)
else:
h3_title = post_view_area.find("h3")
if h3_title:
title_text = h3_title.get_text(strip=True)
else:
strong_title = post_view_area.find("strong")
if strong_title:
title_text = strong_title.get_text(strip=True)
# (C) ๊ทธ๋ž˜๋„ ์—†์œผ๋ฉด <title>์ด๋ผ๋„ ์‚ฌ์šฉ
if not title_text:
head_title = soup.find("title")
if head_title:
title_text = head_title.get_text(strip=True)
else:
title_text = "์ œ๋ชฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
logging.debug(f"์ตœ์ข… ์ถ”์ถœ๋œ ์ œ๋ชฉ: {title_text}")
# --------------------
# ๋ณธ๋ฌธ ์ถ”์ถœ ๋กœ์ง
# --------------------
logging.debug("๋ณธ๋ฌธ ๋‚ด์šฉ ์ถ”์ถœ ์‹œ๋„.")
content_text = None
# (A) ์ƒˆ ์—๋””ํ„ฐ ๊ตฌ์กฐ: div.se-main-container
content_div = soup.find("div", class_="se-main-container")
if content_div:
content_text = content_div.get_text(separator="\n", strip=True)
# (B) ๊ตฌ ์—๋””ํ„ฐ ๊ตฌ์กฐ: div#postViewArea
if not content_text:
old_content_div = soup.find("div", id="postViewArea")
if old_content_div:
content_text = old_content_div.get_text(separator="\n", strip=True)
# (C) ์‹คํŒจ ์‹œ ์•ˆ๋‚ด
if not content_text:
content_text = "๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค."
logging.debug("๋ณธ๋ฌธ ๋‚ด์šฉ ์ถ”์ถœ ์™„๋ฃŒ.")
# ์ œ๋ชฉ + ๋ณธ๋ฌธ ๊ฒฐํ•ฉ
combined_text = f"์ œ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}"
logging.debug("์Šคํฌ๋ž˜ํ•‘ ํ”„๋กœ์„ธ์Šค ์™„๋ฃŒ.")
return combined_text
# Gradio ์ธํ„ฐํŽ˜์ด์Šค
logging.debug("Gradio ์ธํ„ฐํŽ˜์ด์Šค ์ƒ์„ฑ.")
iface = gr.Interface(
fn=scrape_naver_blog,
inputs=gr.Textbox(label="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ", placeholder="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ URL์„ ์ž…๋ ฅํ•˜์„ธ์š”"),
outputs=gr.Textbox(label="์Šคํฌ๋ž˜ํ•‘ ๊ฒฐ๊ณผ", lines=20),
title="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ์Šคํฌ๋ž˜ํ•‘ ๋„๊ตฌ",
description="๋„ค์ด๋ฒ„ ๋ธ”๋กœ๊ทธ ๋งํฌ๋ฅผ ์ž…๋ ฅํ•˜๋ฉด ์ œ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋‚ด์šฉ์„ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค."
)
if __name__ == "__main__":
logging.debug("Gradio ์•ฑ ์‹คํ–‰.")
iface.launch()