Strelema's picture
Update app.py
1740790 verified
Raw History Blame Contribute Delete
11 kB
import streamlit as st
import pandas as pd
import time
import traceback
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
# =====================================================
# PAGE CONFIG
# =====================================================
st.set_page_config(
page_title="Google News Scraper",
page_icon="📰",
layout="centered"
)
# =====================================================
# CLEAN MODERN UI
# =====================================================
st.markdown("""
<style>
#MainMenu {
visibility: hidden;
}
footer {
visibility: hidden;
}
header {
visibility: hidden;
}
.block-container {
max-width: 850px;
padding-top: 2rem;
padding-bottom: 2rem;
}
html, body, [class*="css"] {
font-family: 'Segoe UI', sans-serif;
background-color: #f5f7fa;
}
/* MAIN CARD */
.main-card {
background: white;
padding: 35px;
border-radius: 20px;
box-shadow: 0px 4px 20px rgba(0,0,0,0.06);
}
/* TITLE */
.main-title {
text-align: center;
margin-bottom: 30px;
}
.main-title h1 {
font-size: 40px;
color: #111827;
margin-bottom: 8px;
}
.main-title p {
color: #6b7280;
font-size: 17px;
}
/* TEXT AREA */
textarea {
border-radius: 14px !important;
}
/* INPUT */
.stTextInput input {
border-radius: 12px !important;
}
/* DATE */
.stDateInput > div {
border-radius: 12px !important;
}
/* BUTTON */
.stButton button {
width: 100%;
height: 52px;
border-radius: 14px;
border: none;
background: #2563eb;
color: white;
font-size: 18px;
font-weight: 600;
}
/* DOWNLOAD BUTTON */
.stDownloadButton button {
width: 100%;
height: 52px;
border-radius: 14px;
border: none;
background: #16a34a;
color: white;
font-size: 18px;
font-weight: 600;
}
</style>
""", unsafe_allow_html=True)
# =====================================================
# DRIVER SETUP
# =====================================================
def create_driver():
options = Options()
# Required for Docker / HuggingFace Spaces
options.add_argument("--headless=new")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-gpu")
options.add_argument("--window-size=1920,1080")
# Reduce Chrome crashes
options.add_argument("--disable-software-rasterizer")
options.add_argument("--disable-extensions")
# User agent
options.add_argument(
"--user-agent=Mozilla/5.0 "
"(Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 "
"Chrome/138 Safari/537.36"
)
driver = webdriver.Chrome(
options=options
)
return driver
# =====================================================
# SCRAPER FUNCTION
# =====================================================
def scrape_google_news(
keywords_text,
start_date,
end_date,
max_pages,
excel_name
):
all_data = []
driver = None
try:
driver = create_driver()
RESTART_AFTER = 10
keywords = [
k.strip()
for k in keywords_text.split("\n")
if k.strip()
]
total_keywords = len(keywords)
progress_bar = st.progress(0)
status = st.empty()
for idx, keyword in enumerate(keywords):
# Restart browser periodically
if idx > 0 and idx % RESTART_AFTER == 0:
status.info(
"♻️ Restarting browser to maintain stability..."
)
try:
driver.quit()
except:
pass
time.sleep(3)
driver = create_driver()
progress_bar.progress(
(idx + 1) / total_keywords
)
status.info(
f"🔍 Searching: {keyword}"
)
query = keyword.replace(" ", "+")
url = (
f"https://www.google.com/search?q={query}"
f"&tbm=nws"
f"&tbs=cdr:1,cd_min:{start_date},cd_max:{end_date}"
)
try:
driver.get(url)
# Debugging
print(driver.current_url)
print(driver.title)
time.sleep(4)
except Exception:
status.warning(
f"Connection issue while searching '{keyword}'. "
"Restarting browser..."
)
try:
driver.quit()
except:
pass
driver = create_driver()
continue
for page in range(1, int(max_pages) + 1):
status.info(
f"🔎 Searching: {keyword} | Page {page}"
)
try:
# Retry once if Google is slow
try:
results = driver.find_elements(
By.XPATH,
"//a[.//div[@role='heading']]"
)
except:
time.sleep(4)
try:
results = driver.find_elements(
By.XPATH,
"//a[.//div[@role='heading']]"
)
except:
status.warning(
f"Could not find results for '{keyword}'"
)
break
found_count = 0
for r in results:
try:
title = r.find_element(
By.XPATH,
".//div[@role='heading']"
).text.strip()
link = r.get_attribute("href")
if (
link
and "google.com" not in link
):
all_data.append(
{
"Keyword": keyword,
"Page": page,
"Title": title,
"Link": link,
}
)
found_count += 1
except:
continue
status.info(
f"✅ {keyword} | Page {page}: "
f"Found {found_count} links"
)
try:
next_btn = driver.find_element(
By.XPATH,
"//a[@id='pnnext' or @aria-label='Next']"
)
driver.execute_script(
"arguments[0].scrollIntoView(true);",
next_btn
)
time.sleep(1)
driver.execute_script(
"arguments[0].click();",
next_btn
)
time.sleep(3)
except:
status.info(
f"❌ {keyword}: No more pages available."
)
break
except Exception:
traceback.print_exc()
break
if not all_data:
st.error(
"No data found. "
"Google may have blocked the request or changed the page structure."
)
return None
# ==========================================
# SAVE EXCEL
# ==========================================
df = pd.DataFrame(all_data)
df.drop_duplicates(
subset=["Link"],
inplace=True
)
safe_name = excel_name.strip().replace(
" ",
"_"
)
filename = (
f"{safe_name}_"
f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx"
)
df.to_excel(
filename,
index=False
)
st.success(
f"🎉 Success! {len(df)} unique links collected."
)
return filename
except Exception:
traceback.print_exc()
st.error(
"An unexpected error occurred. "
"Please check the terminal logs."
)
return None
finally:
if driver:
try:
driver.quit()
except:
pass
# =====================================================
# UI
# =====================================================
st.markdown("""
<div class="main-title">
<h1>📰 Google News Scraper</h1>
<p>Google News Extraction Tool with Excel Export</p>
</div>
""", unsafe_allow_html=True)
# st.markdown('<div class="main-card">', unsafe_allow_html=True)
keywords = st.text_area(
"Keywords (One Per Line)",
height=220,
placeholder="""
Chandrashekhar Bawankule OBC
बावनकुळे ओबीसी
Bawankule Kunbi OBC
"""
)
col1, col2 = st.columns(2)
with col1:
start_date = st.date_input(
"Start Date"
)
with col2:
end_date = st.date_input(
"End Date"
)
excel_name = st.text_input(
"Excel File Name",
value="google_news_results"
)
max_pages = st.slider(
"Maximum Pages",
1,
30,
5
)
run_scraper = st.button(
"🚀 Start Scraping"
)
download_placeholder = st.empty()
st.markdown('</div>', unsafe_allow_html=True)
# =====================================================
# RUN SCRAPER
# =====================================================
if run_scraper:
if not keywords.strip():
st.warning("Please enter keywords")
else:
start_date_str = start_date.strftime(
"%m/%d/%Y"
)
end_date_str = end_date.strftime(
"%m/%d/%Y"
)
excel_file = scrape_google_news(
keywords,
start_date_str,
end_date_str,
max_pages,
excel_name
)
if excel_file:
with open(excel_file, "rb") as file:
download_placeholder.download_button(
label="⬇ Download Excel File",
data=file,
file_name=excel_file,
mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)