""" app.py - HuggingFace Space v5 (CSV-Based PDF Extractor) ========================================================= Workflow: 1. User CSV upload karta hai (Source PDF column ke saath) 2. Script har PDF download karti hai 3. SEBI Annexure-A table se data nikalta hai 4. Currency + Timeline normalize hota hai 5. Clean CSV download milta hai """ import os, re, logging import requests, pdfplumber import pandas as pd import gradio as gr from datetime import datetime from io import BytesIO from transformers import pipeline logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) # ══════════════════════════════════════════════ # 🔐 COOKIES - HuggingFace Secrets se # ══════════════════════════════════════════════ COOKIES = { "sessionid": os.environ.get("SESSIONID", ""), "csrftoken": os.environ.get("CSRFTOKEN", ""), } HEADERS = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36" ), "Accept": "application/pdf,*/*", } USD_TO_INR = 85.0 # ══════════════════════════════════════════════ # 🤖 HUGGINGFACE QA MODEL # ══════════════════════════════════════════════ log.info("Loading QA model...") try: hf_qa = pipeline("question-answering", model="deepset/roberta-base-squad2") log.info("✅ Model ready!") except Exception as e: log.warning(f"Model load failed: {e}") hf_qa = None # ══════════════════════════════════════════════ # 💰 CURRENCY NORMALIZER # Koi bhi format → "X.XX Cr" string # ══════════════════════════════════════════════ def normalize_value(raw: str) -> str: """ USD 1,346,210 → 11.44 Cr (USD 1,346,210@₹85) ₹1,587.10 million → 158.71 Cr (₹1587.1Mn) Rs. 890.69 Million → 89.07 Cr (₹890.69Mn) Rs. 17,750 Million → 1775.00 Cr INR 20.58 Crores → 20.58 Cr Rs. 1,827.33 Cr → 1827.33 Cr Rs. 33 Crore Excl. GST → 33.00 Cr """ if not raw or str(raw).strip() in ("", "N/A", "-", "nan"): return "N/A" t = str(raw).strip() # USD m = re.search(r"USD\s*[\$]?\s*([\d,]+(?:\.\d+)?)", t, re.I) if m: v = float(m.group(1).replace(",", "")) cr = round(v * USD_TO_INR / 1e7, 2) return f"{cr:.2f} Cr (USD {v:,.0f}@₹{USD_TO_INR})" # Million / Mn m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:million|mn)\.?\b", t, re.I) if m: v = float(m.group(1).replace(",", "")) cr = round(v / 10, 2) return f"{cr:.2f} Cr (₹{v}Mn)" # Billion / Bn m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:billion|bn)\.?\b", t, re.I) if m: v = float(m.group(1).replace(",", "")) cr = round(v * 100, 2) return f"{cr:.2f} Cr (₹{v}Bn)" # Already in Crore m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:crore|cr\.?|crores)\b", t, re.I) if m: cr = round(float(m.group(1).replace(",", "")), 2) return f"{cr:.2f} Cr" # Lakh m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:lakh|lac)\.?\b", t, re.I) if m: v = float(m.group(1).replace(",", "")) cr = round(v / 100, 4) return f"{cr:.4f} Cr (₹{v}L)" return t # ══════════════════════════════════════════════ # ⏰ TIMELINE NORMALIZER # ══════════════════════════════════════════════ def normalize_timeline(raw: str) -> str: """ 910 Days → 910 Days (~30 Months) 4 (04) Years → 4 Years (~48 Months) 34months → 34 Months 12 weeks → 12 Weeks (~3 Months) 5 years → 5 Years (~60 Months) Within 18 Months → 18 Months Delivery by Oct 2027 → Oct 2027 """ if not raw or str(raw).strip() in ("", "N/A", "-", "nan"): return "N/A" t = str(raw).strip() # Days m = re.search(r"(\d+)\s*days?\b", t, re.I) if m: d = int(m.group(1)) return f"{d} Days (~{round(d/30.4)} Months)" # Weeks m = re.search(r"(\d+)\s*weeks?\b", t, re.I) if m: w = int(m.group(1)) return f"{w} Weeks (~{round(w/4.3)} Months)" # Years m = re.search(r"(\d+(?:\.\d+)?)\s*(?:\(\d+\)\s*)?years?\b", t, re.I) if m: y = float(m.group(1)) mo = round(y * 12) return f"{int(y) if y==int(y) else y} Years (~{mo} Months)" # Months m = re.search(r"(\d+)\s*months?\b(.*?)(?:\.|$)", t, re.I) if m: mo = int(m.group(1)) ctx = m.group(2).strip() suf = f" {ctx}" if ctx and len(ctx) < 50 else "" return f"{mo} Months{suf}" # Specific date m = re.search( r"(?:by|until|before)\s+" r"((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*" r"(?:\s+\d{1,2},?)?\s+\d{4})", t, re.I ) if m: return _short_month(m.group(1).strip()) return t[:80] def _short_month(s: str) -> str: for full, short in { "january":"Jan","february":"Feb","march":"Mar","april":"Apr", "may":"May","june":"Jun","july":"Jul","august":"Aug", "september":"Sep","october":"Oct","november":"Nov","december":"Dec" }.items(): s = re.sub(full, short, s, flags=re.I) return s # ══════════════════════════════════════════════ # 🧹 CUSTOMER CLEANER # ══════════════════════════════════════════════ def clean_customer(val: str) -> str: """ Garbage values hatao: "Domestic Entity\ndomestic" → "" (re-extract needed) "BSE\nLimited..." → "" """ if not val or str(val).strip() in ("N/A", "", "-", "nan"): return "" t = re.sub(r"\s*\n\s*", " ", str(val)).strip() garbage = [ r"^domestic\b", r"^international\b", r"^bse\b", r"^nse\b", r"national stock exchange", r"bombay stock exchange", r"as per point", r"^yes$", r"^no$", ] for g in garbage: if re.search(g, t, re.I): return "" return t[:120] # ══════════════════════════════════════════════ # 📋 SEBI ANNEXURE-A TABLE PARSER # ══════════════════════════════════════════════ def parse_annexure(text: str) -> dict: out = {k: "" for k in ["customer","description","order_type", "dom_intl","timeline_raw","value_raw"]} if not text: return out t = re.sub(r"\s+", " ", text) def grab(patterns): for p in patterns: m = re.search(p, t, re.I) if m: v = m.group(1).strip() if len(v) > 3 and "particular" not in v.lower(): return v[:250] return "" out["customer"] = grab([ r"entity awarding[^:]*?[:\s]+([^\n]{5,120}?)(?:\s*\d+\.|$)", r"awarding the order[^:]*?[:\s]+([^\n]{5,120}?)(?:\s*\d+\.|$)", ]) out["description"] = grab([ r"significant terms[^:]*?[:\s]+([^\n]{10,300}?)(?:\s*\d+\.|$)", r"terms and conditions[^:]*?[:\s]+([^\n]{10,300}?)(?:\s*\d+\.|$)", ]) out["order_type"] = grab([ r"nature of order[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", ]) out["dom_intl"] = grab([ r"domestic or international[^:]*?[:\s]+([^\n]{3,30}?)(?:\s*\d+\.|$)", ]) out["timeline_raw"] = grab([ r"time period[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", r"period by which[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", r"to be executed[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", ]) out["value_raw"] = grab([ r"broad consideration[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", r"size of the order[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", r"consideration or size[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", r"value of.{0,10}order[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", r"total.{0,10}(?:value|amount|cost)[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)", ]) return out # ══════════════════════════════════════════════ # 📥 PDF TEXT EXTRACT # ══════════════════════════════════════════════ def get_pdf_text(url: str) -> str: if not url or str(url).strip() in ("", "nan"): return "" try: resp = requests.get( str(url).strip(), headers={**HEADERS, "Referer": "https://www.bseindia.com/"}, timeout=60 ) resp.raise_for_status() parts = [] with pdfplumber.open(BytesIO(resp.content)) as pdf: for i, page in enumerate(pdf.pages): t = page.extract_text() if t: parts.append(t) if i >= 4: break return "\n".join(parts).strip() except Exception as e: log.warning(f"PDF error: {e}") return "" # ══════════════════════════════════════════════ # 🎯 SINGLE ROW PROCESSOR # CSV ki ek row ko PDF se enrich karta hai # ══════════════════════════════════════════════ def process_row(row: pd.Series) -> dict: pdf_url = str(row.get("Source PDF", "")).strip() # Current values check karo cur_cust = clean_customer(str(row.get("Customer", ""))) cur_val = str(row.get("New Order Value (Cr)", "")).strip() cur_tl = str(row.get("Timeline", "")).strip() cur_type = str(row.get("Order Type", "")).strip() # Kya PDF se nikalna hai? need_cust = not cur_cust need_val = cur_val in ("N/A","","nan") or not re.search(r"Cr|USD", cur_val) need_tl = cur_tl in ("N/A","","nan") need_type = cur_type in ("N/A","","nan") # PDF sirf tabhi fetch karo jab zaroorat ho pdf_text = "" if any([need_cust, need_val, need_tl, need_type]) and pdf_url: pdf_text = get_pdf_text(pdf_url) ann = parse_annexure(pdf_text) if pdf_text else {} # ── CUSTOMER ── if need_cust: cust = clean_customer(ann.get("customer", "")) if not cust and hf_qa and pdf_text: try: ctx = " ".join(pdf_text.split())[:1500] a = hf_qa({"question": "Who is the customer or entity awarding this order?", "context": ctx}) if a["score"] > 0.15: cust = clean_customer(a["answer"]) except: pass final_cust = cust or cur_cust or "N/A" else: final_cust = cur_cust # ── VALUE ── raw_val = ann.get("value_raw", "") if need_val else cur_val final_value = normalize_value(raw_val or cur_val) # ── TIMELINE ── raw_tl = ann.get("timeline_raw", "") if need_tl else cur_tl final_tl = normalize_timeline(raw_tl or cur_tl) # ── ORDER TYPE ── if need_type: ot = ann.get("order_type", "") if not ot: combined = pdf_text + str(row.get("Filing Title","")) for name in ["Letter of Award","Letter of Intent","Work Order", "EPC Contract","Repeat Order","Export Order", "Supply Order","Contract Award","Purchase Order"]: if name.lower() in combined.lower(): ot = name break final_type = ot or "N/A" else: final_type = cur_type desc = ann.get("description","") or str(row.get("Filing Title","")) return { "Stock Name": row.get("Stock Name",""), "Exchange": row.get("Exchange",""), "Date Order Received": row.get("Date Order Received",""), "Customer": final_cust, "Order Type": final_type, "Order Description": desc[:200] if desc else "N/A", "Order Value (INR Cr)": final_value, "Value Raw": (raw_val or cur_val)[:100], "Timeline": final_tl, "Domestic/Intl": ann.get("dom_intl","N/A") or "N/A", "Source PDF": pdf_url, "Filing Title": row.get("Filing Title",""), "Screener Date": row.get("Screener Date",""), "Processed At": datetime.now().strftime("%Y-%m-%d %H:%M"), } # ══════════════════════════════════════════════ # 🚀 GRADIO FUNCTION 1: CSV Upload + Process # ══════════════════════════════════════════════ def process_csv(csv_file, progress=gr.Progress()): if csv_file is None: return None, "❌ Pehle CSV file upload karo!", None try: df = pd.read_csv(csv_file.name) except Exception as e: return None, f"❌ CSV read error: {e}", None if "Source PDF" not in df.columns: return None, "❌ 'Source PDF' column CSV mein nahi hai!", None total = len(df) results = [] logs = [f"✅ CSV loaded: {total} rows", ""] for i, (_, row) in enumerate(df.iterrows()): progress((i+1)/total, desc=f"Row {i+1}/{total}: {str(row.get('Stock Name',''))[:25]}") try: enriched = process_row(row) results.append(enriched) logs.append( f"[{i+1:>3}/{total}] ✅ {enriched['Stock Name'][:22]:<22} | " f"{enriched['Order Value (INR Cr)']:<18} | " f"{enriched['Timeline']:<18} | " f"{enriched['Customer'][:25]}" ) except Exception as e: logs.append(f"[{i+1:>3}/{total}] ❌ {row.get('Stock Name','?')}: {e}") results.append({"Stock Name":row.get("Stock Name","?"), "Source PDF":row.get("Source PDF",""), "Error":str(e)}) out_df = pd.DataFrame(results) out_path = "enriched_orders.csv" out_df.to_csv(out_path, index=False, encoding="utf-8-sig") logs.append(f"\n🏁 Complete! {len(results)} rows → {out_path}") return out_df, "\n".join(logs), out_path # ══════════════════════════════════════════════ # 🚀 GRADIO FUNCTION 2: Fresh Scraper # ══════════════════════════════════════════════ def run_fresh_scraper(pages: int, progress=gr.Progress()): from bs4 import BeautifulSoup SCREENER_URL = ( "https://www.screener.in/full-text-search/" "?q=-%22Commissioner%22+-%22tax%22+-%22gst%22" "+%22order+received%22+or+%22award+of+order%22" "+or+%22Notification+of+Award%22+or+%22letter+of+intent%22" "+or+%22large+order%22+or+%22Order+for+Procurement%22" "+or+%22Awarding+of+order%22+or+%22bagged+an+order+%22" "+or+%22Letter+of+Award%22+or+%22repeat+order%22" "+or+%22additional+order%22+or+%22Contract+Award%22%29" "&page={page}" ) if not COOKIES["sessionid"]: return None, "❌ SESSIONID Secret nahi! Space Settings → Secrets mein add karo.", None all_records, logs = [], [] for page in range(1, int(pages)+1): progress(page/pages, desc=f"Page {page}/{int(pages)}...") logs.append(f"📄 Page {page}/{int(pages)}...") try: resp = requests.get( SCREENER_URL.format(page=page), headers={**HEADERS, "Referer":"https://www.screener.in/"}, cookies=COOKIES, timeout=30 ) if "/login" in resp.url: logs.append("❌ Cookies expire! Update karo."); break soup = BeautifulSoup(resp.text, "html.parser") cards = soup.select("div.margin-top-20.margin-bottom-36") filings = [] for card in cards: link = card.select_one("a[href*='/company/']") if not link: continue span = link.select_one("span.hover-link") stock = span.get_text(strip=True) if span else link.get_text(strip=True) pd_ = card.select_one("div.font-size-17") if not pd_: continue pl = pd_.find("a", href=True) if not pl: continue url = pl["href"] exch = "BSE" if "bseindia" in url else "NSE" snip = card.select_one("div.ink-700.font-size-16") snippet = snip.get_text(" ",strip=True)[:200] if snip else "" dd = card.select_one("div.margin-top-4.ink-700") date_str, doc_type = "", "Unknown" if dd: parts = [p.strip() for p in dd.get_text(" ",strip=True).split("-")] if len(parts)>=2: doc_type, date_str = parts[0], parts[-1] if doc_type == "Announcement": filings.append({"stock":stock,"exchange":exch,"pdf_url":url, "title":pl.get_text(strip=True), "snippet":snippet,"date":date_str}) logs.append(f" {len(filings)} Announcements mile") for f in filings: pdf_text = get_pdf_text(f["pdf_url"]) ann = parse_annexure(pdf_text) record = { "Stock Name": f["stock"], "Exchange": f["exchange"], "Date Order Received": f["date"], "Customer": clean_customer(ann.get("customer","")) or "N/A", "Order Type": ann.get("order_type","") or "N/A", "Order Description": ann.get("description","")[:200] or "N/A", "Order Value (INR Cr)": normalize_value(ann.get("value_raw","")), "Value Raw": ann.get("value_raw","")[:100], "Timeline": normalize_timeline(ann.get("timeline_raw","")), "Domestic/Intl": ann.get("dom_intl","N/A"), "Source PDF": f["pdf_url"], "Filing Title": f["title"], "Scraped At": datetime.now().strftime("%Y-%m-%d %H:%M"), } all_records.append(record) logs.append(f" 📌 {record['Stock Name'][:20]:<20} | " f"{record['Order Value (INR Cr)']:<16} | " f"{record['Customer'][:25]}") except Exception as e: logs.append(f" ❌ Page {page}: {e}") if not all_records: return None, "❌ Koi data nahi mila.", None out_df = pd.DataFrame(all_records) out_path = "fresh_orders.csv" out_df.to_csv(out_path, index=False, encoding="utf-8-sig") logs.append(f"\n🏁 Done! {len(all_records)} records → {out_path}") return out_df, "\n".join(logs), out_path # ══════════════════════════════════════════════ # 🎨 GRADIO UI # ══════════════════════════════════════════════ with gr.Blocks(title="BSE/NSE Order Extractor v5", theme=gr.themes.Soft()) as demo: gr.Markdown(""" # 📊 BSE / NSE Order Extractor v5 ### CSV Upload → PDF Extract → Auto Normalize → Download ✨ | Mode | Kab use karein | |------|----------------| | 📁 **CSV Upload** | Already downloaded CSV ko enrich karna ho | | 🌐 **Fresh Scrape** | Screener.in se naya data chahiye (Cookies required) | """) with gr.Tabs(): # ── TAB 1: CSV UPLOAD ── with gr.TabItem("📁 Mode 1: CSV Upload (Recommended)"): gr.Markdown(""" **Steps:** 1. Apni CSV upload karo (jo pehle download ki thi) 2. "Process" button dabaao — PDFs auto-download honge 3. Clean enriched CSV download karo! **Kya fix hoga automatically:** | Problem | Fix | |---------|-----| | `USD 30` | `2.55 Cr (USD 30@₹85)` | | `₹890 million` | `89.00 Cr (₹890Mn)` | | `34months` | `34 Months` | | `12 weeks` | `12 Weeks (~3 Months)` | | `Domestic Entity` (wrong customer) | PDF se actual customer | | `N/A` order type | PDF se actual type | """) csv_in = gr.File(label="📂 CSV File Upload karo", file_types=[".csv"]) proc_btn = gr.Button("🚀 Process CSV → Extract PDFs", variant="primary", size="lg") gr.Markdown("---") stat1 = gr.Textbox(label="📋 Live Progress", lines=14, interactive=False) tbl1 = gr.Dataframe(label="📊 Enriched Preview", wrap=True) dl1 = gr.File(label="⬇️ Download Enriched CSV") proc_btn.click(fn=process_csv, inputs=[csv_in], outputs=[tbl1, stat1, dl1]) # ── TAB 2: FRESH SCRAPE ── with gr.TabItem("🌐 Mode 2: Fresh Scrape"): gr.Markdown(""" **Cookies Setup (ek baar):** ``` Space → Settings → Repository secrets SESSIONID = browser se copy karo CSRFTOKEN = browser se copy karo ``` """) pages_in = gr.Slider(minimum=1, maximum=20, value=3, step=1, label="📄 Kitne Pages?") scrape_btn = gr.Button("🚀 Fresh Scrape Karo", variant="primary", size="lg") gr.Markdown("---") stat2 = gr.Textbox(label="📋 Live Progress", lines=14, interactive=False) tbl2 = gr.Dataframe(label="📊 Fresh Data", wrap=True) dl2 = gr.File(label="⬇️ Download Fresh CSV") scrape_btn.click(fn=run_fresh_scraper, inputs=[pages_in], outputs=[tbl2, stat2, dl2]) gr.Markdown(""" --- **Output CSV Columns:** `Stock Name` | `Exchange` | `Customer` | `Order Type` | `Order Description` | `Order Value (INR Cr)` | `Timeline` | `Domestic/Intl` | `Date` | `Source PDF` 💱 **Auto-Convert:** USD → INR Cr | Million → Cr | Billion → Cr | Lakh → Cr ⏰ **Timeline Normalize:** Days/Weeks/Years/Months/Dates → Standard format """) if __name__ == "__main__": demo.launch()