Spaces:
Sleeping
Sleeping
Download app.py from Thestrelema/google_link_scraper: direct link, hf CLI and curl.
- Browser
- Download file 11 kB
-
https://huggingface.co/spaces/Thestrelema/google_link_scraper/resolve/main/app.py
- Command line
-
hf download hf://spaces/Thestrelema/google_link_scraper/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Thestrelema/google_link_scraper/resolve/main/app.py
11 kB
| import streamlit as st | |
| import pandas as pd | |
| import time | |
| import traceback | |
| from datetime import datetime | |
| from selenium import webdriver | |
| from selenium.webdriver.common.by import By | |
| from selenium.webdriver.chrome.options import Options | |
| # ===================================================== | |
| # PAGE CONFIG | |
| # ===================================================== | |
| st.set_page_config( | |
| page_title="Google News Scraper", | |
| page_icon="📰", | |
| layout="centered" | |
| ) | |
| # ===================================================== | |
| # CLEAN MODERN UI | |
| # ===================================================== | |
| st.markdown(""" | |
| <style> | |
| #MainMenu { | |
| visibility: hidden; | |
| } | |
| footer { | |
| visibility: hidden; | |
| } | |
| header { | |
| visibility: hidden; | |
| } | |
| .block-container { | |
| max-width: 850px; | |
| padding-top: 2rem; | |
| padding-bottom: 2rem; | |
| } | |
| html, body, [class*="css"] { | |
| font-family: 'Segoe UI', sans-serif; | |
| background-color: #f5f7fa; | |
| } | |
| /* MAIN CARD */ | |
| .main-card { | |
| background: white; | |
| padding: 35px; | |
| border-radius: 20px; | |
| box-shadow: 0px 4px 20px rgba(0,0,0,0.06); | |
| } | |
| /* TITLE */ | |
| .main-title { | |
| text-align: center; | |
| margin-bottom: 30px; | |
| } | |
| .main-title h1 { | |
| font-size: 40px; | |
| color: #111827; | |
| margin-bottom: 8px; | |
| } | |
| .main-title p { | |
| color: #6b7280; | |
| font-size: 17px; | |
| } | |
| /* TEXT AREA */ | |
| textarea { | |
| border-radius: 14px !important; | |
| } | |
| /* INPUT */ | |
| .stTextInput input { | |
| border-radius: 12px !important; | |
| } | |
| /* DATE */ | |
| .stDateInput > div { | |
| border-radius: 12px !important; | |
| } | |
| /* BUTTON */ | |
| .stButton button { | |
| width: 100%; | |
| height: 52px; | |
| border-radius: 14px; | |
| border: none; | |
| background: #2563eb; | |
| color: white; | |
| font-size: 18px; | |
| font-weight: 600; | |
| } | |
| /* DOWNLOAD BUTTON */ | |
| .stDownloadButton button { | |
| width: 100%; | |
| height: 52px; | |
| border-radius: 14px; | |
| border: none; | |
| background: #16a34a; | |
| color: white; | |
| font-size: 18px; | |
| font-weight: 600; | |
| } | |
| </style> | |
| """, unsafe_allow_html=True) | |
| # ===================================================== | |
| # DRIVER SETUP | |
| # ===================================================== | |
| def create_driver(): | |
| options = Options() | |
| # Required for Docker / HuggingFace Spaces | |
| options.add_argument("--headless=new") | |
| options.add_argument("--no-sandbox") | |
| options.add_argument("--disable-dev-shm-usage") | |
| options.add_argument("--disable-gpu") | |
| options.add_argument("--window-size=1920,1080") | |
| # Reduce Chrome crashes | |
| options.add_argument("--disable-software-rasterizer") | |
| options.add_argument("--disable-extensions") | |
| # User agent | |
| options.add_argument( | |
| "--user-agent=Mozilla/5.0 " | |
| "(Windows NT 10.0; Win64; x64) " | |
| "AppleWebKit/537.36 " | |
| "Chrome/138 Safari/537.36" | |
| ) | |
| driver = webdriver.Chrome( | |
| options=options | |
| ) | |
| return driver | |
| # ===================================================== | |
| # SCRAPER FUNCTION | |
| # ===================================================== | |
| def scrape_google_news( | |
| keywords_text, | |
| start_date, | |
| end_date, | |
| max_pages, | |
| excel_name | |
| ): | |
| all_data = [] | |
| driver = None | |
| try: | |
| driver = create_driver() | |
| RESTART_AFTER = 10 | |
| keywords = [ | |
| k.strip() | |
| for k in keywords_text.split("\n") | |
| if k.strip() | |
| ] | |
| total_keywords = len(keywords) | |
| progress_bar = st.progress(0) | |
| status = st.empty() | |
| for idx, keyword in enumerate(keywords): | |
| # Restart browser periodically | |
| if idx > 0 and idx % RESTART_AFTER == 0: | |
| status.info( | |
| "♻️ Restarting browser to maintain stability..." | |
| ) | |
| try: | |
| driver.quit() | |
| except: | |
| pass | |
| time.sleep(3) | |
| driver = create_driver() | |
| progress_bar.progress( | |
| (idx + 1) / total_keywords | |
| ) | |
| status.info( | |
| f"🔍 Searching: {keyword}" | |
| ) | |
| query = keyword.replace(" ", "+") | |
| url = ( | |
| f"https://www.google.com/search?q={query}" | |
| f"&tbm=nws" | |
| f"&tbs=cdr:1,cd_min:{start_date},cd_max:{end_date}" | |
| ) | |
| try: | |
| driver.get(url) | |
| # Debugging | |
| print(driver.current_url) | |
| print(driver.title) | |
| time.sleep(4) | |
| except Exception: | |
| status.warning( | |
| f"Connection issue while searching '{keyword}'. " | |
| "Restarting browser..." | |
| ) | |
| try: | |
| driver.quit() | |
| except: | |
| pass | |
| driver = create_driver() | |
| continue | |
| for page in range(1, int(max_pages) + 1): | |
| status.info( | |
| f"🔎 Searching: {keyword} | Page {page}" | |
| ) | |
| try: | |
| # Retry once if Google is slow | |
| try: | |
| results = driver.find_elements( | |
| By.XPATH, | |
| "//a[.//div[@role='heading']]" | |
| ) | |
| except: | |
| time.sleep(4) | |
| try: | |
| results = driver.find_elements( | |
| By.XPATH, | |
| "//a[.//div[@role='heading']]" | |
| ) | |
| except: | |
| status.warning( | |
| f"Could not find results for '{keyword}'" | |
| ) | |
| break | |
| found_count = 0 | |
| for r in results: | |
| try: | |
| title = r.find_element( | |
| By.XPATH, | |
| ".//div[@role='heading']" | |
| ).text.strip() | |
| link = r.get_attribute("href") | |
| if ( | |
| link | |
| and "google.com" not in link | |
| ): | |
| all_data.append( | |
| { | |
| "Keyword": keyword, | |
| "Page": page, | |
| "Title": title, | |
| "Link": link, | |
| } | |
| ) | |
| found_count += 1 | |
| except: | |
| continue | |
| status.info( | |
| f"✅ {keyword} | Page {page}: " | |
| f"Found {found_count} links" | |
| ) | |
| try: | |
| next_btn = driver.find_element( | |
| By.XPATH, | |
| "//a[@id='pnnext' or @aria-label='Next']" | |
| ) | |
| driver.execute_script( | |
| "arguments[0].scrollIntoView(true);", | |
| next_btn | |
| ) | |
| time.sleep(1) | |
| driver.execute_script( | |
| "arguments[0].click();", | |
| next_btn | |
| ) | |
| time.sleep(3) | |
| except: | |
| status.info( | |
| f"❌ {keyword}: No more pages available." | |
| ) | |
| break | |
| except Exception: | |
| traceback.print_exc() | |
| break | |
| if not all_data: | |
| st.error( | |
| "No data found. " | |
| "Google may have blocked the request or changed the page structure." | |
| ) | |
| return None | |
| # ========================================== | |
| # SAVE EXCEL | |
| # ========================================== | |
| df = pd.DataFrame(all_data) | |
| df.drop_duplicates( | |
| subset=["Link"], | |
| inplace=True | |
| ) | |
| safe_name = excel_name.strip().replace( | |
| " ", | |
| "_" | |
| ) | |
| filename = ( | |
| f"{safe_name}_" | |
| f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" | |
| ) | |
| df.to_excel( | |
| filename, | |
| index=False | |
| ) | |
| st.success( | |
| f"🎉 Success! {len(df)} unique links collected." | |
| ) | |
| return filename | |
| except Exception: | |
| traceback.print_exc() | |
| st.error( | |
| "An unexpected error occurred. " | |
| "Please check the terminal logs." | |
| ) | |
| return None | |
| finally: | |
| if driver: | |
| try: | |
| driver.quit() | |
| except: | |
| pass | |
| # ===================================================== | |
| # UI | |
| # ===================================================== | |
| st.markdown(""" | |
| <div class="main-title"> | |
| <h1>📰 Google News Scraper</h1> | |
| <p>Google News Extraction Tool with Excel Export</p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # st.markdown('<div class="main-card">', unsafe_allow_html=True) | |
| keywords = st.text_area( | |
| "Keywords (One Per Line)", | |
| height=220, | |
| placeholder=""" | |
| Chandrashekhar Bawankule OBC | |
| बावनकुळे ओबीसी | |
| Bawankule Kunbi OBC | |
| """ | |
| ) | |
| col1, col2 = st.columns(2) | |
| with col1: | |
| start_date = st.date_input( | |
| "Start Date" | |
| ) | |
| with col2: | |
| end_date = st.date_input( | |
| "End Date" | |
| ) | |
| excel_name = st.text_input( | |
| "Excel File Name", | |
| value="google_news_results" | |
| ) | |
| max_pages = st.slider( | |
| "Maximum Pages", | |
| 1, | |
| 30, | |
| 5 | |
| ) | |
| run_scraper = st.button( | |
| "🚀 Start Scraping" | |
| ) | |
| download_placeholder = st.empty() | |
| st.markdown('</div>', unsafe_allow_html=True) | |
| # ===================================================== | |
| # RUN SCRAPER | |
| # ===================================================== | |
| if run_scraper: | |
| if not keywords.strip(): | |
| st.warning("Please enter keywords") | |
| else: | |
| start_date_str = start_date.strftime( | |
| "%m/%d/%Y" | |
| ) | |
| end_date_str = end_date.strftime( | |
| "%m/%d/%Y" | |
| ) | |
| excel_file = scrape_google_news( | |
| keywords, | |
| start_date_str, | |
| end_date_str, | |
| max_pages, | |
| excel_name | |
| ) | |
| if excel_file: | |
| with open(excel_file, "rb") as file: | |
| download_placeholder.download_button( | |
| label="⬇ Download Excel File", | |
| data=file, | |
| file_name=excel_file, | |
| mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" | |
| ) |