Download scraper.py from Ai-Bharti/pdf_data: direct link, hf CLI and curl.
- Browser
- Download file 3.92 kB
-
https://huggingface.co/spaces/Ai-Bharti/pdf_data/resolve/main/scraper.py
- Command line
-
hf download hf://spaces/Ai-Bharti/pdf_data/scraper.py
-
curl -L -o scraper.py https://huggingface.co/spaces/Ai-Bharti/pdf_data/resolve/main/scraper.py
3.92 kB
| import requests | |
| from bs4 import BeautifulSoup | |
| import os | |
| # β EXACT URL jo aapne diya - isme se page number change hoga only | |
| # Jaise ek bus ka route fixed hota hai - sirf stop number badalta hai! | |
| BASE_SEARCH_URL = ( | |
| "https://www.screener.in/full-text-search/" | |
| "?q=-%22Commissioner%22+-%22tax%22+-%22gst%22" | |
| "+%22order+received%22+or+%22award+of+order%22" | |
| "+or+%22Notification+of+Award%22+or+%22letter+of+intent%22" | |
| "+or+%22large+order%22+or+%22Order+for+Procurement%22" | |
| "+or+%22Awarding+of+order%22+or+%22bagged+an+order+%22" | |
| "+or+%22Letter+of+Award%22+or+%22repeat+order%22" | |
| "+or+%22additional+order%22+or+%22Contract+Award%22%29" | |
| ) | |
| def get_pdf_urls_from_screener(page_num=1, cookies=None): | |
| """ | |
| Screener.in ke EXACT search URL se BSE PDF links nikalte hain. | |
| page_num = URL mein &page=17 wala number | |
| cookies = Screener login ki chabi π | |
| """ | |
| # Page number add karo URL ke end mein | |
| # Jaise: ...&page=1, ...&page=2, ...&page=17 | |
| full_url = f"{BASE_SEARCH_URL}&page={page_num}" | |
| print(f"π Fetching URL: {full_url[:80]}...") | |
| headers = { | |
| # Browser jaisa dikhna zaroori - warna Screener block kar dega | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36", | |
| "Referer": "https://www.screener.in/", | |
| "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", | |
| } | |
| try: | |
| session = requests.Session() | |
| # β Cookies Priority System: | |
| # 1οΈβ£ Manually di hui cookies (UI se) | |
| # 2οΈβ£ HuggingFace Secrets se auto-load | |
| # 3οΈβ£ Kuch nahi = Guest mode | |
| if cookies and cookies.get("sessionid"): | |
| session.cookies.update(cookies) | |
| print("β Manual cookies use ho rahi hain") | |
| elif os.environ.get("SCREENER_SESSION_ID"): | |
| env_cookies = { | |
| "sessionid": os.environ.get("SCREENER_SESSION_ID", ""), | |
| "csrftoken": os.environ.get("SCREENER_CSRF_TOKEN", ""), | |
| } | |
| session.cookies.update(env_cookies) | |
| print("π HuggingFace Secrets se cookies load hui!") | |
| else: | |
| print("β οΈ Cookies nahi mili - guest mode mein chalega") | |
| # β EXACT URL pe GET request bhejo | |
| response = session.get(full_url, headers=headers, timeout=20) | |
| response.raise_for_status() | |
| print(f"π‘ Response status: {response.status_code}") | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| pdf_links = [] | |
| company_names = [] | |
| # Page mein saare BSE PDF links dhundo | |
| # Jaise library mein sirf 'science' section ki books dhundna | |
| for a_tag in soup.find_all("a", href=True): | |
| link = a_tag["href"] | |
| # Sirf BSE ke PDF links chahiye | |
| if "bseindia.com" in link and ".pdf" in link.lower(): | |
| full_link = link if link.startswith("http") else "https:" + link | |
| # Duplicate links skip karo | |
| if full_link not in pdf_links: | |
| pdf_links.append(full_link) | |
| # Company name dhundne ki koshish | |
| parent = a_tag.find_parent( | |
| class_=lambda c: c and "company" in c.lower() | |
| ) | |
| company_names.append( | |
| parent.get_text(strip=True)[:50] if parent else "Unknown Company" | |
| ) | |
| print(f"β Page {page_num}: {len(pdf_links)} PDF links mile!") | |
| return list(zip(company_names, pdf_links)) | |
| except requests.exceptions.HTTPError as e: | |
| print(f"β HTTP Error {e.response.status_code}: Cookies expire ho gayi hain?") | |
| return [] | |
| except requests.exceptions.RequestException as e: | |
| print(f"β Network error: {e}") | |
| return [] | |