dataSraping / app.py
Ai-Bharti's picture
Update app.py
527b3a8 verified
Raw History Blame Contribute Delete
23.7 kB
"""
app.py - HuggingFace Space v5 (CSV-Based PDF Extractor)
=========================================================
Workflow:
1. User CSV upload karta hai (Source PDF column ke saath)
2. Script har PDF download karti hai
3. SEBI Annexure-A table se data nikalta hai
4. Currency + Timeline normalize hota hai
5. Clean CSV download milta hai
"""
import os, re, logging
import requests, pdfplumber
import pandas as pd
import gradio as gr
from datetime import datetime
from io import BytesIO
from transformers import pipeline
logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)
# ══════════════════════════════════════════════
# πŸ” COOKIES - HuggingFace Secrets se
# ══════════════════════════════════════════════
COOKIES = {
"sessionid": os.environ.get("SESSIONID", ""),
"csrftoken": os.environ.get("CSRFTOKEN", ""),
}
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36"
),
"Accept": "application/pdf,*/*",
}
USD_TO_INR = 85.0
# ══════════════════════════════════════════════
# πŸ€– HUGGINGFACE QA MODEL
# ══════════════════════════════════════════════
log.info("Loading QA model...")
try:
hf_qa = pipeline("question-answering", model="deepset/roberta-base-squad2")
log.info("βœ… Model ready!")
except Exception as e:
log.warning(f"Model load failed: {e}")
hf_qa = None
# ══════════════════════════════════════════════
# πŸ’° CURRENCY NORMALIZER
# Koi bhi format β†’ "X.XX Cr" string
# ══════════════════════════════════════════════
def normalize_value(raw: str) -> str:
"""
USD 1,346,210 β†’ 11.44 Cr (USD 1,346,210@β‚Ή85)
β‚Ή1,587.10 million β†’ 158.71 Cr (β‚Ή1587.1Mn)
Rs. 890.69 Million β†’ 89.07 Cr (β‚Ή890.69Mn)
Rs. 17,750 Million β†’ 1775.00 Cr
INR 20.58 Crores β†’ 20.58 Cr
Rs. 1,827.33 Cr β†’ 1827.33 Cr
Rs. 33 Crore Excl. GST β†’ 33.00 Cr
"""
if not raw or str(raw).strip() in ("", "N/A", "-", "nan"):
return "N/A"
t = str(raw).strip()
# USD
m = re.search(r"USD\s*[\$]?\s*([\d,]+(?:\.\d+)?)", t, re.I)
if m:
v = float(m.group(1).replace(",", ""))
cr = round(v * USD_TO_INR / 1e7, 2)
return f"{cr:.2f} Cr (USD {v:,.0f}@β‚Ή{USD_TO_INR})"
# Million / Mn
m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:million|mn)\.?\b", t, re.I)
if m:
v = float(m.group(1).replace(",", ""))
cr = round(v / 10, 2)
return f"{cr:.2f} Cr (β‚Ή{v}Mn)"
# Billion / Bn
m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:billion|bn)\.?\b", t, re.I)
if m:
v = float(m.group(1).replace(",", ""))
cr = round(v * 100, 2)
return f"{cr:.2f} Cr (β‚Ή{v}Bn)"
# Already in Crore
m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:crore|cr\.?|crores)\b", t, re.I)
if m:
cr = round(float(m.group(1).replace(",", "")), 2)
return f"{cr:.2f} Cr"
# Lakh
m = re.search(r"([\d,]+(?:\.\d+)?)\s*(?:lakh|lac)\.?\b", t, re.I)
if m:
v = float(m.group(1).replace(",", ""))
cr = round(v / 100, 4)
return f"{cr:.4f} Cr (β‚Ή{v}L)"
return t
# ══════════════════════════════════════════════
# ⏰ TIMELINE NORMALIZER
# ══════════════════════════════════════════════
def normalize_timeline(raw: str) -> str:
"""
910 Days β†’ 910 Days (~30 Months)
4 (04) Years β†’ 4 Years (~48 Months)
34months β†’ 34 Months
12 weeks β†’ 12 Weeks (~3 Months)
5 years β†’ 5 Years (~60 Months)
Within 18 Months β†’ 18 Months
Delivery by Oct 2027 β†’ Oct 2027
"""
if not raw or str(raw).strip() in ("", "N/A", "-", "nan"):
return "N/A"
t = str(raw).strip()
# Days
m = re.search(r"(\d+)\s*days?\b", t, re.I)
if m:
d = int(m.group(1))
return f"{d} Days (~{round(d/30.4)} Months)"
# Weeks
m = re.search(r"(\d+)\s*weeks?\b", t, re.I)
if m:
w = int(m.group(1))
return f"{w} Weeks (~{round(w/4.3)} Months)"
# Years
m = re.search(r"(\d+(?:\.\d+)?)\s*(?:\(\d+\)\s*)?years?\b", t, re.I)
if m:
y = float(m.group(1))
mo = round(y * 12)
return f"{int(y) if y==int(y) else y} Years (~{mo} Months)"
# Months
m = re.search(r"(\d+)\s*months?\b(.*?)(?:\.|$)", t, re.I)
if m:
mo = int(m.group(1))
ctx = m.group(2).strip()
suf = f" {ctx}" if ctx and len(ctx) < 50 else ""
return f"{mo} Months{suf}"
# Specific date
m = re.search(
r"(?:by|until|before)\s+"
r"((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*"
r"(?:\s+\d{1,2},?)?\s+\d{4})",
t, re.I
)
if m:
return _short_month(m.group(1).strip())
return t[:80]
def _short_month(s: str) -> str:
for full, short in {
"january":"Jan","february":"Feb","march":"Mar","april":"Apr",
"may":"May","june":"Jun","july":"Jul","august":"Aug",
"september":"Sep","october":"Oct","november":"Nov","december":"Dec"
}.items():
s = re.sub(full, short, s, flags=re.I)
return s
# ══════════════════════════════════════════════
# 🧹 CUSTOMER CLEANER
# ══════════════════════════════════════════════
def clean_customer(val: str) -> str:
"""
Garbage values hatao:
"Domestic Entity\ndomestic" β†’ "" (re-extract needed)
"BSE\nLimited..." β†’ ""
"""
if not val or str(val).strip() in ("N/A", "", "-", "nan"):
return ""
t = re.sub(r"\s*\n\s*", " ", str(val)).strip()
garbage = [
r"^domestic\b", r"^international\b",
r"^bse\b", r"^nse\b",
r"national stock exchange",
r"bombay stock exchange",
r"as per point", r"^yes$", r"^no$",
]
for g in garbage:
if re.search(g, t, re.I):
return ""
return t[:120]
# ══════════════════════════════════════════════
# πŸ“‹ SEBI ANNEXURE-A TABLE PARSER
# ══════════════════════════════════════════════
def parse_annexure(text: str) -> dict:
out = {k: "" for k in ["customer","description","order_type",
"dom_intl","timeline_raw","value_raw"]}
if not text:
return out
t = re.sub(r"\s+", " ", text)
def grab(patterns):
for p in patterns:
m = re.search(p, t, re.I)
if m:
v = m.group(1).strip()
if len(v) > 3 and "particular" not in v.lower():
return v[:250]
return ""
out["customer"] = grab([
r"entity awarding[^:]*?[:\s]+([^\n]{5,120}?)(?:\s*\d+\.|$)",
r"awarding the order[^:]*?[:\s]+([^\n]{5,120}?)(?:\s*\d+\.|$)",
])
out["description"] = grab([
r"significant terms[^:]*?[:\s]+([^\n]{10,300}?)(?:\s*\d+\.|$)",
r"terms and conditions[^:]*?[:\s]+([^\n]{10,300}?)(?:\s*\d+\.|$)",
])
out["order_type"] = grab([
r"nature of order[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
])
out["dom_intl"] = grab([
r"domestic or international[^:]*?[:\s]+([^\n]{3,30}?)(?:\s*\d+\.|$)",
])
out["timeline_raw"] = grab([
r"time period[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
r"period by which[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
r"to be executed[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
])
out["value_raw"] = grab([
r"broad consideration[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
r"size of the order[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
r"consideration or size[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
r"value of.{0,10}order[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
r"total.{0,10}(?:value|amount|cost)[^:]*?[:\s]+([^\n]{5,200}?)(?:\s*\d+\.|$)",
])
return out
# ══════════════════════════════════════════════
# πŸ“₯ PDF TEXT EXTRACT
# ══════════════════════════════════════════════
def get_pdf_text(url: str) -> str:
if not url or str(url).strip() in ("", "nan"):
return ""
try:
resp = requests.get(
str(url).strip(),
headers={**HEADERS, "Referer": "https://www.bseindia.com/"},
timeout=60
)
resp.raise_for_status()
parts = []
with pdfplumber.open(BytesIO(resp.content)) as pdf:
for i, page in enumerate(pdf.pages):
t = page.extract_text()
if t: parts.append(t)
if i >= 4: break
return "\n".join(parts).strip()
except Exception as e:
log.warning(f"PDF error: {e}")
return ""
# ══════════════════════════════════════════════
# 🎯 SINGLE ROW PROCESSOR
# CSV ki ek row ko PDF se enrich karta hai
# ══════════════════════════════════════════════
def process_row(row: pd.Series) -> dict:
pdf_url = str(row.get("Source PDF", "")).strip()
# Current values check karo
cur_cust = clean_customer(str(row.get("Customer", "")))
cur_val = str(row.get("New Order Value (Cr)", "")).strip()
cur_tl = str(row.get("Timeline", "")).strip()
cur_type = str(row.get("Order Type", "")).strip()
# Kya PDF se nikalna hai?
need_cust = not cur_cust
need_val = cur_val in ("N/A","","nan") or not re.search(r"Cr|USD", cur_val)
need_tl = cur_tl in ("N/A","","nan")
need_type = cur_type in ("N/A","","nan")
# PDF sirf tabhi fetch karo jab zaroorat ho
pdf_text = ""
if any([need_cust, need_val, need_tl, need_type]) and pdf_url:
pdf_text = get_pdf_text(pdf_url)
ann = parse_annexure(pdf_text) if pdf_text else {}
# ── CUSTOMER ──
if need_cust:
cust = clean_customer(ann.get("customer", ""))
if not cust and hf_qa and pdf_text:
try:
ctx = " ".join(pdf_text.split())[:1500]
a = hf_qa({"question": "Who is the customer or entity awarding this order?",
"context": ctx})
if a["score"] > 0.15:
cust = clean_customer(a["answer"])
except:
pass
final_cust = cust or cur_cust or "N/A"
else:
final_cust = cur_cust
# ── VALUE ──
raw_val = ann.get("value_raw", "") if need_val else cur_val
final_value = normalize_value(raw_val or cur_val)
# ── TIMELINE ──
raw_tl = ann.get("timeline_raw", "") if need_tl else cur_tl
final_tl = normalize_timeline(raw_tl or cur_tl)
# ── ORDER TYPE ──
if need_type:
ot = ann.get("order_type", "")
if not ot:
combined = pdf_text + str(row.get("Filing Title",""))
for name in ["Letter of Award","Letter of Intent","Work Order",
"EPC Contract","Repeat Order","Export Order",
"Supply Order","Contract Award","Purchase Order"]:
if name.lower() in combined.lower():
ot = name
break
final_type = ot or "N/A"
else:
final_type = cur_type
desc = ann.get("description","") or str(row.get("Filing Title",""))
return {
"Stock Name": row.get("Stock Name",""),
"Exchange": row.get("Exchange",""),
"Date Order Received": row.get("Date Order Received",""),
"Customer": final_cust,
"Order Type": final_type,
"Order Description": desc[:200] if desc else "N/A",
"Order Value (INR Cr)": final_value,
"Value Raw": (raw_val or cur_val)[:100],
"Timeline": final_tl,
"Domestic/Intl": ann.get("dom_intl","N/A") or "N/A",
"Source PDF": pdf_url,
"Filing Title": row.get("Filing Title",""),
"Screener Date": row.get("Screener Date",""),
"Processed At": datetime.now().strftime("%Y-%m-%d %H:%M"),
}
# ══════════════════════════════════════════════
# πŸš€ GRADIO FUNCTION 1: CSV Upload + Process
# ══════════════════════════════════════════════
def process_csv(csv_file, progress=gr.Progress()):
if csv_file is None:
return None, "❌ Pehle CSV file upload karo!", None
try:
df = pd.read_csv(csv_file.name)
except Exception as e:
return None, f"❌ CSV read error: {e}", None
if "Source PDF" not in df.columns:
return None, "❌ 'Source PDF' column CSV mein nahi hai!", None
total = len(df)
results = []
logs = [f"βœ… CSV loaded: {total} rows", ""]
for i, (_, row) in enumerate(df.iterrows()):
progress((i+1)/total, desc=f"Row {i+1}/{total}: {str(row.get('Stock Name',''))[:25]}")
try:
enriched = process_row(row)
results.append(enriched)
logs.append(
f"[{i+1:>3}/{total}] βœ… {enriched['Stock Name'][:22]:<22} | "
f"{enriched['Order Value (INR Cr)']:<18} | "
f"{enriched['Timeline']:<18} | "
f"{enriched['Customer'][:25]}"
)
except Exception as e:
logs.append(f"[{i+1:>3}/{total}] ❌ {row.get('Stock Name','?')}: {e}")
results.append({"Stock Name":row.get("Stock Name","?"),
"Source PDF":row.get("Source PDF",""), "Error":str(e)})
out_df = pd.DataFrame(results)
out_path = "enriched_orders.csv"
out_df.to_csv(out_path, index=False, encoding="utf-8-sig")
logs.append(f"\n🏁 Complete! {len(results)} rows β†’ {out_path}")
return out_df, "\n".join(logs), out_path
# ══════════════════════════════════════════════
# πŸš€ GRADIO FUNCTION 2: Fresh Scraper
# ══════════════════════════════════════════════
def run_fresh_scraper(pages: int, progress=gr.Progress()):
from bs4 import BeautifulSoup
SCREENER_URL = (
"https://www.screener.in/full-text-search/"
"?q=-%22Commissioner%22+-%22tax%22+-%22gst%22"
"+%22order+received%22+or+%22award+of+order%22"
"+or+%22Notification+of+Award%22+or+%22letter+of+intent%22"
"+or+%22large+order%22+or+%22Order+for+Procurement%22"
"+or+%22Awarding+of+order%22+or+%22bagged+an+order+%22"
"+or+%22Letter+of+Award%22+or+%22repeat+order%22"
"+or+%22additional+order%22+or+%22Contract+Award%22%29"
"&page={page}"
)
if not COOKIES["sessionid"]:
return None, "❌ SESSIONID Secret nahi! Space Settings β†’ Secrets mein add karo.", None
all_records, logs = [], []
for page in range(1, int(pages)+1):
progress(page/pages, desc=f"Page {page}/{int(pages)}...")
logs.append(f"πŸ“„ Page {page}/{int(pages)}...")
try:
resp = requests.get(
SCREENER_URL.format(page=page),
headers={**HEADERS, "Referer":"https://www.screener.in/"},
cookies=COOKIES, timeout=30
)
if "/login" in resp.url:
logs.append("❌ Cookies expire! Update karo."); break
soup = BeautifulSoup(resp.text, "html.parser")
cards = soup.select("div.margin-top-20.margin-bottom-36")
filings = []
for card in cards:
link = card.select_one("a[href*='/company/']")
if not link: continue
span = link.select_one("span.hover-link")
stock = span.get_text(strip=True) if span else link.get_text(strip=True)
pd_ = card.select_one("div.font-size-17")
if not pd_: continue
pl = pd_.find("a", href=True)
if not pl: continue
url = pl["href"]
exch = "BSE" if "bseindia" in url else "NSE"
snip = card.select_one("div.ink-700.font-size-16")
snippet = snip.get_text(" ",strip=True)[:200] if snip else ""
dd = card.select_one("div.margin-top-4.ink-700")
date_str, doc_type = "", "Unknown"
if dd:
parts = [p.strip() for p in dd.get_text(" ",strip=True).split("-")]
if len(parts)>=2: doc_type, date_str = parts[0], parts[-1]
if doc_type == "Announcement":
filings.append({"stock":stock,"exchange":exch,"pdf_url":url,
"title":pl.get_text(strip=True),
"snippet":snippet,"date":date_str})
logs.append(f" {len(filings)} Announcements mile")
for f in filings:
pdf_text = get_pdf_text(f["pdf_url"])
ann = parse_annexure(pdf_text)
record = {
"Stock Name": f["stock"],
"Exchange": f["exchange"],
"Date Order Received": f["date"],
"Customer": clean_customer(ann.get("customer","")) or "N/A",
"Order Type": ann.get("order_type","") or "N/A",
"Order Description": ann.get("description","")[:200] or "N/A",
"Order Value (INR Cr)": normalize_value(ann.get("value_raw","")),
"Value Raw": ann.get("value_raw","")[:100],
"Timeline": normalize_timeline(ann.get("timeline_raw","")),
"Domestic/Intl": ann.get("dom_intl","N/A"),
"Source PDF": f["pdf_url"],
"Filing Title": f["title"],
"Scraped At": datetime.now().strftime("%Y-%m-%d %H:%M"),
}
all_records.append(record)
logs.append(f" πŸ“Œ {record['Stock Name'][:20]:<20} | "
f"{record['Order Value (INR Cr)']:<16} | "
f"{record['Customer'][:25]}")
except Exception as e:
logs.append(f" ❌ Page {page}: {e}")
if not all_records:
return None, "❌ Koi data nahi mila.", None
out_df = pd.DataFrame(all_records)
out_path = "fresh_orders.csv"
out_df.to_csv(out_path, index=False, encoding="utf-8-sig")
logs.append(f"\n🏁 Done! {len(all_records)} records β†’ {out_path}")
return out_df, "\n".join(logs), out_path
# ══════════════════════════════════════════════
# 🎨 GRADIO UI
# ══════════════════════════════════════════════
with gr.Blocks(title="BSE/NSE Order Extractor v5", theme=gr.themes.Soft()) as demo:
gr.Markdown("""
# πŸ“Š BSE / NSE Order Extractor v5
### CSV Upload β†’ PDF Extract β†’ Auto Normalize β†’ Download ✨
| Mode | Kab use karein |
|------|----------------|
| πŸ“ **CSV Upload** | Already downloaded CSV ko enrich karna ho |
| 🌐 **Fresh Scrape** | Screener.in se naya data chahiye (Cookies required) |
""")
with gr.Tabs():
# ── TAB 1: CSV UPLOAD ──
with gr.TabItem("πŸ“ Mode 1: CSV Upload (Recommended)"):
gr.Markdown("""
**Steps:**
1. Apni CSV upload karo (jo pehle download ki thi)
2. "Process" button dabaao β€” PDFs auto-download honge
3. Clean enriched CSV download karo!
**Kya fix hoga automatically:**
| Problem | Fix |
|---------|-----|
| `USD 30` | `2.55 Cr (USD 30@β‚Ή85)` |
| `β‚Ή890 million` | `89.00 Cr (β‚Ή890Mn)` |
| `34months` | `34 Months` |
| `12 weeks` | `12 Weeks (~3 Months)` |
| `Domestic Entity` (wrong customer) | PDF se actual customer |
| `N/A` order type | PDF se actual type |
""")
csv_in = gr.File(label="πŸ“‚ CSV File Upload karo", file_types=[".csv"])
proc_btn = gr.Button("πŸš€ Process CSV β†’ Extract PDFs", variant="primary", size="lg")
gr.Markdown("---")
stat1 = gr.Textbox(label="πŸ“‹ Live Progress", lines=14, interactive=False)
tbl1 = gr.Dataframe(label="πŸ“Š Enriched Preview", wrap=True)
dl1 = gr.File(label="⬇️ Download Enriched CSV")
proc_btn.click(fn=process_csv, inputs=[csv_in], outputs=[tbl1, stat1, dl1])
# ── TAB 2: FRESH SCRAPE ──
with gr.TabItem("🌐 Mode 2: Fresh Scrape"):
gr.Markdown("""
**Cookies Setup (ek baar):**
```
Space β†’ Settings β†’ Repository secrets
SESSIONID = browser se copy karo
CSRFTOKEN = browser se copy karo
```
""")
pages_in = gr.Slider(minimum=1, maximum=20, value=3, step=1,
label="πŸ“„ Kitne Pages?")
scrape_btn = gr.Button("πŸš€ Fresh Scrape Karo", variant="primary", size="lg")
gr.Markdown("---")
stat2 = gr.Textbox(label="πŸ“‹ Live Progress", lines=14, interactive=False)
tbl2 = gr.Dataframe(label="πŸ“Š Fresh Data", wrap=True)
dl2 = gr.File(label="⬇️ Download Fresh CSV")
scrape_btn.click(fn=run_fresh_scraper, inputs=[pages_in], outputs=[tbl2, stat2, dl2])
gr.Markdown("""
---
**Output CSV Columns:** `Stock Name` | `Exchange` | `Customer` | `Order Type` |
`Order Description` | `Order Value (INR Cr)` | `Timeline` | `Domestic/Intl` | `Date` | `Source PDF`
πŸ’± **Auto-Convert:** USD β†’ INR Cr | Million β†’ Cr | Billion β†’ Cr | Lakh β†’ Cr
⏰ **Timeline Normalize:** Days/Weeks/Years/Months/Dates β†’ Standard format
""")
if __name__ == "__main__":
demo.launch()