Download app.py from Sangwon78/test1: direct link, hf CLI and curl.
- Browser
- Download file 7.3 kB
-
https://huggingface.co/spaces/Sangwon78/test1/resolve/main/app.py
- Command line
-
hf download hf://spaces/Sangwon78/test1/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Sangwon78/test1/resolve/main/app.py
7.3 kB
| import requests | |
| import random | |
| import time | |
| import logging | |
| from bs4 import BeautifulSoup | |
| import gradio as gr | |
| from urllib.parse import urlparse, urlunparse | |
| """ | |
| [๊ธฐ๋ณธ์ฌํญ] | |
| # ํ๊ตญ์ด๋ก ์ค๋ช | |
| # ํ๊น ํ์ด์ค์ ๊ทธ๋ผ๋์ค ํ๊ฒฝ์์ ๋์๋๋ ์คํ์ด์ค | |
| # ์์ฒญํ ์ฌํญ์ ์ฝ๋๋ฅผ ์์์ ์ผ๋ก ์ถ์/์ญ์ ํ์ง ๋ง๊ณ ์ ์ฒด์ฝ๋๋ฅผ ์ถ๋ ฅ | |
| [์์ฒญ์ฌํญ] | |
| 1) ๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ๋งํฌ๋ฅผ ์ ๋ ฅํ๋ฉด, ํด๋น ๋งํฌ์ ์ ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋ด์ฉ์ ์คํฌ๋ํ | |
| 2) ์ถ๋ ฅ์ ์ ๋ชฉ๊ณผ ๋ณธ๋ฌธ๋ด์ฉ์ด ํ ์คํธ์ฐฝ์ ํ ๋ฒ์ ๋์ค๋๋ก | |
| 3) ํ๊ฒ URL ์์: | |
| - ๊ธฐ๋ณธ: https://blog.naver.com/acoloje/223783254325 | |
| 4) ๋ฐ์ดํฐ HTML ๊ตฌ์กฐ(์ ๋ชฉ): | |
| <div class="se-module se-module-text se-title-text"> | |
| <p class="se-text-paragraph se-text-paragraph-align-" style="" id="SE-d4c7e111-1d97-40cf-a490-eb188c4b7f04"> | |
| <span style="" class="se-fs- se-ff-" id="SE-b62ca798-fd52-11ef-8346-2d7ed694d63c"><!-- -->์๋ฐ๋ฅ์ M ๋งํฌ๊ฐ ์์ด์?<!-- --></span> | |
| </p> | |
| </div> | |
| 5) ๋ฐ์ดํฐHTML ๊ตฌ์กฐ(๋ณธ๋ฌธ): | |
| <div class="se-main-container"> | |
| ... | |
| <div class="se-component se-text se-l-default" ...> | |
| ... | |
| <p class="se-text-paragraph ...">๋ณธ๋ฌธ ๋ด์ฉ</p> | |
| ... | |
| </div> | |
| ... | |
| </div> | |
| [์ฐจ๋จ๋ฐฉ์ง ์ ์ฉ์๋ฃ] | |
| 1. Header ์ค์ ํ๊ธฐ (User-Agent, Referer ๋ฑ) | |
| 2. User-Agent ๋๋ค ์ค์ | |
| 3. Referer ์ค์ (๋ค์ด๋ฒ ๋ชจ๋ฐ์ผ ํ์ด์ง์์ ์จ ๊ฒ์ฒ๋ผ ๋ณด์ด๋๋ก) | |
| 4. ๋๋ค ๋๋ ์ด (1~3์ด) | |
| """ | |
| # ๋๋ฒ๊น ๋ก๊ทธ ์ค์ (UI์๋ ๋ณด์ด์ง ์๊ณ debug.log ํ์ผ์ ๊ธฐ๋ก) | |
| logging.basicConfig( | |
| filename='debug.log', | |
| level=logging.DEBUG, | |
| format='%(asctime)s - %(levelname)s - %(message)s' | |
| ) | |
| logging.debug("์ ํ๋ฆฌ์ผ์ด์ ์์.") | |
| # ์ฌ๋ฌ ๋ธ๋ผ์ฐ์ ์ User-Agent ๋ฌธ์์ด ๋ฆฌ์คํธ | |
| USER_AGENTS = [ | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36", | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0", | |
| "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15", | |
| ] | |
| def get_random_user_agent(): | |
| user_agent = random.choice(USER_AGENTS) | |
| logging.debug(f"์ ํ๋ User-Agent: {user_agent}") | |
| return user_agent | |
| def convert_to_mobile_url(url): | |
| """ | |
| PC ๋ฒ์ blog.naver.com ์ฃผ์๋ฅผ m.blog.naver.com์ผ๋ก ๋ณํํ์ฌ | |
| ๋ชจ๋ฐ์ผ ํ์ด์ง๋ฅผ ์์ฒญํ ์ ์๋๋ก ํจ. | |
| """ | |
| parsed = urlparse(url) | |
| if "blog.naver.com" in parsed.netloc and "m.blog.naver.com" not in parsed.netloc: | |
| new_netloc = parsed.netloc.replace("blog.naver.com", "m.blog.naver.com") | |
| new_url = urlunparse((parsed.scheme, new_netloc, parsed.path, parsed.params, parsed.query, parsed.fragment)) | |
| logging.debug(f"๋ชจ๋ฐ์ผ ์ฃผ์๋ก ๋ณํ: {new_url}") | |
| return new_url | |
| return url | |
| def scrape_naver_blog(url): | |
| logging.debug("์คํฌ๋ํ ํ๋ก์ธ์ค ์์.") | |
| # 1) URL์ ๋ชจ๋ฐ์ผ ๋ฒ์ ์ผ๋ก ๋ณํ | |
| mobile_url = convert_to_mobile_url(url) | |
| logging.debug(f"์ต์ข ์์ฒญํ URL: {mobile_url}") | |
| # 2) ์์ฒญ ์ ๋๋ค ๋๋ ์ด (1~3์ด) | |
| delay = random.uniform(1, 3) | |
| logging.debug(f"์์ฒญ ์ {delay:.2f}์ด ๋๊ธฐ.") | |
| time.sleep(delay) | |
| # 3) HTTP ํค๋ ์ค์ (User-Agent, Referer ๋ฑ) | |
| headers = { | |
| "User-Agent": get_random_user_agent(), | |
| "Referer": "https://m.blog.naver.com", | |
| "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7" | |
| } | |
| logging.debug(f"์์ฒญ ํค๋: {headers}") | |
| # 4) GET ์์ฒญ | |
| try: | |
| logging.debug(f"URL๋ก GET ์์ฒญ ์ ์ก: {mobile_url}") | |
| response = requests.get(mobile_url, headers=headers, timeout=10) | |
| logging.debug(f"์๋ต ์ํ ์ฝ๋: {response.status_code}") | |
| except Exception as e: | |
| logging.error(f"HTTP ์์ฒญ ์ค ์๋ฌ ๋ฐ์: {e}") | |
| return f"HTTP ์์ฒญ ์ค ์๋ฌ ๋ฐ์: {e}" | |
| if response.status_code != 200: | |
| logging.error("200์ด ์๋ ์ํ ์ฝ๋ ์์ .") | |
| return f"HTTP ์์ฒญ ์คํจ, ์ํ ์ฝ๋: {response.status_code}" | |
| # 5) HTML ํ์ฑ | |
| logging.debug("HTML ํ์ฑ ์์.") | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # -------------------- | |
| # ์ ๋ชฉ ์ถ์ถ ๋ก์ง | |
| # -------------------- | |
| logging.debug("์ ๋ชฉ ์ถ์ถ ์๋.") | |
| # (A) ์ ์๋ํฐ ๊ตฌ์กฐ | |
| title_text = None | |
| title_div = soup.find("div", class_="se-module se-module-text se-title-text") | |
| if title_div: | |
| p_tag = title_div.find("p") | |
| if p_tag: | |
| span_tag = p_tag.find("span") | |
| if span_tag: | |
| title_text = span_tag.get_text(strip=True) | |
| # (B) ๊ตฌ ์๋ํฐ ๊ตฌ์กฐ: #postViewArea ๋ด๋ถ ๋ฑ | |
| if not title_text: | |
| post_view_area = soup.find("div", id="postViewArea") | |
| if post_view_area: | |
| candidate_title = post_view_area.find("span", class_="pcol1 itemSubjectBoldfont") | |
| if candidate_title: | |
| title_text = candidate_title.get_text(strip=True) | |
| else: | |
| h3_title = post_view_area.find("h3") | |
| if h3_title: | |
| title_text = h3_title.get_text(strip=True) | |
| else: | |
| strong_title = post_view_area.find("strong") | |
| if strong_title: | |
| title_text = strong_title.get_text(strip=True) | |
| # (C) ๊ทธ๋๋ ์์ผ๋ฉด <title>์ด๋ผ๋ ์ฌ์ฉ | |
| if not title_text: | |
| head_title = soup.find("title") | |
| if head_title: | |
| title_text = head_title.get_text(strip=True) | |
| else: | |
| title_text = "์ ๋ชฉ์ ์ฐพ์ ์ ์์ต๋๋ค." | |
| logging.debug(f"์ต์ข ์ถ์ถ๋ ์ ๋ชฉ: {title_text}") | |
| # -------------------- | |
| # ๋ณธ๋ฌธ ์ถ์ถ ๋ก์ง | |
| # -------------------- | |
| logging.debug("๋ณธ๋ฌธ ๋ด์ฉ ์ถ์ถ ์๋.") | |
| content_text = None | |
| # (A) ์ ์๋ํฐ ๊ตฌ์กฐ: div.se-main-container | |
| content_div = soup.find("div", class_="se-main-container") | |
| if content_div: | |
| content_text = content_div.get_text(separator="\n", strip=True) | |
| # (B) ๊ตฌ ์๋ํฐ ๊ตฌ์กฐ: div#postViewArea | |
| if not content_text: | |
| old_content_div = soup.find("div", id="postViewArea") | |
| if old_content_div: | |
| content_text = old_content_div.get_text(separator="\n", strip=True) | |
| # (C) ์คํจ ์ ์๋ด | |
| if not content_text: | |
| content_text = "๋ณธ๋ฌธ ๋ด์ฉ์ ์ฐพ์ ์ ์์ต๋๋ค." | |
| logging.debug("๋ณธ๋ฌธ ๋ด์ฉ ์ถ์ถ ์๋ฃ.") | |
| # ์ ๋ชฉ + ๋ณธ๋ฌธ ๊ฒฐํฉ | |
| combined_text = f"์ ๋ชฉ: {title_text}\n\n๋ณธ๋ฌธ:\n{content_text}" | |
| logging.debug("์คํฌ๋ํ ํ๋ก์ธ์ค ์๋ฃ.") | |
| return combined_text | |
| # Gradio ์ธํฐํ์ด์ค | |
| logging.debug("Gradio ์ธํฐํ์ด์ค ์์ฑ.") | |
| iface = gr.Interface( | |
| fn=scrape_naver_blog, | |
| inputs=gr.Textbox(label="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ๋งํฌ", placeholder="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ URL์ ์ ๋ ฅํ์ธ์"), | |
| outputs=gr.Textbox(label="์คํฌ๋ํ ๊ฒฐ๊ณผ", lines=20), | |
| title="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ์คํฌ๋ํ ๋๊ตฌ", | |
| description="๋ค์ด๋ฒ ๋ธ๋ก๊ทธ ๋งํฌ๋ฅผ ์ ๋ ฅํ๋ฉด ์ ๋ชฉ๊ณผ ๋ณธ๋ฌธ ๋ด์ฉ์ ์ถ์ถํฉ๋๋ค." | |
| ) | |
| if __name__ == "__main__": | |
| logging.debug("Gradio ์ฑ ์คํ.") | |
| iface.launch() | |