pdf_data / scraper.py
Ai-Bharti's picture
Upload scraper.py
d92c887 verified
Raw History Blame Contribute Delete
3.92 kB
import requests
from bs4 import BeautifulSoup
import os
# βœ… EXACT URL jo aapne diya - isme se page number change hoga only
# Jaise ek bus ka route fixed hota hai - sirf stop number badalta hai!
BASE_SEARCH_URL = (
"https://www.screener.in/full-text-search/"
"?q=-%22Commissioner%22+-%22tax%22+-%22gst%22"
"+%22order+received%22+or+%22award+of+order%22"
"+or+%22Notification+of+Award%22+or+%22letter+of+intent%22"
"+or+%22large+order%22+or+%22Order+for+Procurement%22"
"+or+%22Awarding+of+order%22+or+%22bagged+an+order+%22"
"+or+%22Letter+of+Award%22+or+%22repeat+order%22"
"+or+%22additional+order%22+or+%22Contract+Award%22%29"
)
def get_pdf_urls_from_screener(page_num=1, cookies=None):
"""
Screener.in ke EXACT search URL se BSE PDF links nikalte hain.
page_num = URL mein &page=17 wala number
cookies = Screener login ki chabi πŸ”‘
"""
# Page number add karo URL ke end mein
# Jaise: ...&page=1, ...&page=2, ...&page=17
full_url = f"{BASE_SEARCH_URL}&page={page_num}"
print(f"🌐 Fetching URL: {full_url[:80]}...")
headers = {
# Browser jaisa dikhna zaroori - warna Screener block kar dega
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
"Referer": "https://www.screener.in/",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}
try:
session = requests.Session()
# βœ… Cookies Priority System:
# 1️⃣ Manually di hui cookies (UI se)
# 2️⃣ HuggingFace Secrets se auto-load
# 3️⃣ Kuch nahi = Guest mode
if cookies and cookies.get("sessionid"):
session.cookies.update(cookies)
print("βœ… Manual cookies use ho rahi hain")
elif os.environ.get("SCREENER_SESSION_ID"):
env_cookies = {
"sessionid": os.environ.get("SCREENER_SESSION_ID", ""),
"csrftoken": os.environ.get("SCREENER_CSRF_TOKEN", ""),
}
session.cookies.update(env_cookies)
print("πŸ”’ HuggingFace Secrets se cookies load hui!")
else:
print("⚠️ Cookies nahi mili - guest mode mein chalega")
# βœ… EXACT URL pe GET request bhejo
response = session.get(full_url, headers=headers, timeout=20)
response.raise_for_status()
print(f"πŸ“‘ Response status: {response.status_code}")
soup = BeautifulSoup(response.text, "html.parser")
pdf_links = []
company_names = []
# Page mein saare BSE PDF links dhundo
# Jaise library mein sirf 'science' section ki books dhundna
for a_tag in soup.find_all("a", href=True):
link = a_tag["href"]
# Sirf BSE ke PDF links chahiye
if "bseindia.com" in link and ".pdf" in link.lower():
full_link = link if link.startswith("http") else "https:" + link
# Duplicate links skip karo
if full_link not in pdf_links:
pdf_links.append(full_link)
# Company name dhundne ki koshish
parent = a_tag.find_parent(
class_=lambda c: c and "company" in c.lower()
)
company_names.append(
parent.get_text(strip=True)[:50] if parent else "Unknown Company"
)
print(f"βœ… Page {page_num}: {len(pdf_links)} PDF links mile!")
return list(zip(company_names, pdf_links))
except requests.exceptions.HTTPError as e:
print(f"❌ HTTP Error {e.response.status_code}: Cookies expire ho gayi hain?")
return []
except requests.exceptions.RequestException as e:
print(f"❌ Network error: {e}")
return []