📰 Google News Scraper
Google News Extraction Tool with Excel Export
import streamlit as st import pandas as pd import time import traceback from datetime import datetime from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options # ===================================================== # PAGE CONFIG # ===================================================== st.set_page_config( page_title="Google News Scraper", page_icon="📰", layout="centered" ) # ===================================================== # CLEAN MODERN UI # ===================================================== st.markdown(""" """, unsafe_allow_html=True) # ===================================================== # DRIVER SETUP # ===================================================== def create_driver(): options = Options() # Required for Docker / HuggingFace Spaces options.add_argument("--headless=new") options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-gpu") options.add_argument("--window-size=1920,1080") # Reduce Chrome crashes options.add_argument("--disable-software-rasterizer") options.add_argument("--disable-extensions") # User agent options.add_argument( "--user-agent=Mozilla/5.0 " "(Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 " "Chrome/138 Safari/537.36" ) driver = webdriver.Chrome( options=options ) return driver # ===================================================== # SCRAPER FUNCTION # ===================================================== def scrape_google_news( keywords_text, start_date, end_date, max_pages, excel_name ): all_data = [] driver = None try: driver = create_driver() RESTART_AFTER = 10 keywords = [ k.strip() for k in keywords_text.split("\n") if k.strip() ] total_keywords = len(keywords) progress_bar = st.progress(0) status = st.empty() for idx, keyword in enumerate(keywords): # Restart browser periodically if idx > 0 and idx % RESTART_AFTER == 0: status.info( "♻️ Restarting browser to maintain stability..." ) try: driver.quit() except: pass time.sleep(3) driver = create_driver() progress_bar.progress( (idx + 1) / total_keywords ) status.info( f"🔍 Searching: {keyword}" ) query = keyword.replace(" ", "+") url = ( f"https://www.google.com/search?q={query}" f"&tbm=nws" f"&tbs=cdr:1,cd_min:{start_date},cd_max:{end_date}" ) try: driver.get(url) # Debugging print(driver.current_url) print(driver.title) time.sleep(4) except Exception: status.warning( f"Connection issue while searching '{keyword}'. " "Restarting browser..." ) try: driver.quit() except: pass driver = create_driver() continue for page in range(1, int(max_pages) + 1): status.info( f"🔎 Searching: {keyword} | Page {page}" ) try: # Retry once if Google is slow try: results = driver.find_elements( By.XPATH, "//a[.//div[@role='heading']]" ) except: time.sleep(4) try: results = driver.find_elements( By.XPATH, "//a[.//div[@role='heading']]" ) except: status.warning( f"Could not find results for '{keyword}'" ) break found_count = 0 for r in results: try: title = r.find_element( By.XPATH, ".//div[@role='heading']" ).text.strip() link = r.get_attribute("href") if ( link and "google.com" not in link ): all_data.append( { "Keyword": keyword, "Page": page, "Title": title, "Link": link, } ) found_count += 1 except: continue status.info( f"✅ {keyword} | Page {page}: " f"Found {found_count} links" ) try: next_btn = driver.find_element( By.XPATH, "//a[@id='pnnext' or @aria-label='Next']" ) driver.execute_script( "arguments[0].scrollIntoView(true);", next_btn ) time.sleep(1) driver.execute_script( "arguments[0].click();", next_btn ) time.sleep(3) except: status.info( f"❌ {keyword}: No more pages available." ) break except Exception: traceback.print_exc() break if not all_data: st.error( "No data found. " "Google may have blocked the request or changed the page structure." ) return None # ========================================== # SAVE EXCEL # ========================================== df = pd.DataFrame(all_data) df.drop_duplicates( subset=["Link"], inplace=True ) safe_name = excel_name.strip().replace( " ", "_" ) filename = ( f"{safe_name}_" f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" ) df.to_excel( filename, index=False ) st.success( f"🎉 Success! {len(df)} unique links collected." ) return filename except Exception: traceback.print_exc() st.error( "An unexpected error occurred. " "Please check the terminal logs." ) return None finally: if driver: try: driver.quit() except: pass # ===================================================== # UI # ===================================================== st.markdown("""
Google News Extraction Tool with Excel Export