import streamlit as st import pandas as pd import time import traceback from datetime import datetime from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options # ===================================================== # PAGE CONFIG # ===================================================== st.set_page_config( page_title="Google News Scraper", page_icon="📰", layout="centered" ) # ===================================================== # CLEAN MODERN UI # ===================================================== st.markdown(""" """, unsafe_allow_html=True) # ===================================================== # DRIVER SETUP # ===================================================== def create_driver(): options = Options() # Required for Docker / HuggingFace Spaces options.add_argument("--headless=new") options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-gpu") options.add_argument("--window-size=1920,1080") # Reduce Chrome crashes options.add_argument("--disable-software-rasterizer") options.add_argument("--disable-extensions") # User agent options.add_argument( "--user-agent=Mozilla/5.0 " "(Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 " "Chrome/138 Safari/537.36" ) driver = webdriver.Chrome( options=options ) return driver # ===================================================== # SCRAPER FUNCTION # ===================================================== def scrape_google_news( keywords_text, start_date, end_date, max_pages, excel_name ): all_data = [] driver = None try: driver = create_driver() RESTART_AFTER = 10 keywords = [ k.strip() for k in keywords_text.split("\n") if k.strip() ] total_keywords = len(keywords) progress_bar = st.progress(0) status = st.empty() for idx, keyword in enumerate(keywords): # Restart browser periodically if idx > 0 and idx % RESTART_AFTER == 0: status.info( "♻️ Restarting browser to maintain stability..." ) try: driver.quit() except: pass time.sleep(3) driver = create_driver() progress_bar.progress( (idx + 1) / total_keywords ) status.info( f"🔍 Searching: {keyword}" ) query = keyword.replace(" ", "+") url = ( f"https://www.google.com/search?q={query}" f"&tbm=nws" f"&tbs=cdr:1,cd_min:{start_date},cd_max:{end_date}" ) try: driver.get(url) # Debugging print(driver.current_url) print(driver.title) time.sleep(4) except Exception: status.warning( f"Connection issue while searching '{keyword}'. " "Restarting browser..." ) try: driver.quit() except: pass driver = create_driver() continue for page in range(1, int(max_pages) + 1): status.info( f"🔎 Searching: {keyword} | Page {page}" ) try: # Retry once if Google is slow try: results = driver.find_elements( By.XPATH, "//a[.//div[@role='heading']]" ) except: time.sleep(4) try: results = driver.find_elements( By.XPATH, "//a[.//div[@role='heading']]" ) except: status.warning( f"Could not find results for '{keyword}'" ) break found_count = 0 for r in results: try: title = r.find_element( By.XPATH, ".//div[@role='heading']" ).text.strip() link = r.get_attribute("href") if ( link and "google.com" not in link ): all_data.append( { "Keyword": keyword, "Page": page, "Title": title, "Link": link, } ) found_count += 1 except: continue status.info( f"✅ {keyword} | Page {page}: " f"Found {found_count} links" ) try: next_btn = driver.find_element( By.XPATH, "//a[@id='pnnext' or @aria-label='Next']" ) driver.execute_script( "arguments[0].scrollIntoView(true);", next_btn ) time.sleep(1) driver.execute_script( "arguments[0].click();", next_btn ) time.sleep(3) except: status.info( f"❌ {keyword}: No more pages available." ) break except Exception: traceback.print_exc() break if not all_data: st.error( "No data found. " "Google may have blocked the request or changed the page structure." ) return None # ========================================== # SAVE EXCEL # ========================================== df = pd.DataFrame(all_data) df.drop_duplicates( subset=["Link"], inplace=True ) safe_name = excel_name.strip().replace( " ", "_" ) filename = ( f"{safe_name}_" f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" ) df.to_excel( filename, index=False ) st.success( f"🎉 Success! {len(df)} unique links collected." ) return filename except Exception: traceback.print_exc() st.error( "An unexpected error occurred. " "Please check the terminal logs." ) return None finally: if driver: try: driver.quit() except: pass # ===================================================== # UI # ===================================================== st.markdown("""

📰 Google News Scraper

Google News Extraction Tool with Excel Export

""", unsafe_allow_html=True) # st.markdown('
', unsafe_allow_html=True) keywords = st.text_area( "Keywords (One Per Line)", height=220, placeholder=""" Chandrashekhar Bawankule OBC बावनकुळे ओबीसी Bawankule Kunbi OBC """ ) col1, col2 = st.columns(2) with col1: start_date = st.date_input( "Start Date" ) with col2: end_date = st.date_input( "End Date" ) excel_name = st.text_input( "Excel File Name", value="google_news_results" ) max_pages = st.slider( "Maximum Pages", 1, 30, 5 ) run_scraper = st.button( "🚀 Start Scraping" ) download_placeholder = st.empty() st.markdown('
', unsafe_allow_html=True) # ===================================================== # RUN SCRAPER # ===================================================== if run_scraper: if not keywords.strip(): st.warning("Please enter keywords") else: start_date_str = start_date.strftime( "%m/%d/%Y" ) end_date_str = end_date.strftime( "%m/%d/%Y" ) excel_file = scrape_google_news( keywords, start_date_str, end_date_str, max_pages, excel_name ) if excel_file: with open(excel_file, "rb") as file: download_placeholder.download_button( label="⬇ Download Excel File", data=file, file_name=excel_file, mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" )