Spaces:
Sleeping
Sleeping
Download data_normalizer.py from Ai-Bharti/dataSraping: direct link, hf CLI and curl.
- Browser
- Download file 19.6 kB
-
https://huggingface.co/spaces/Ai-Bharti/dataSraping/resolve/main/data_normalizer.py
- Command line
-
hf download hf://spaces/Ai-Bharti/dataSraping/data_normalizer.py
-
curl -L -o data_normalizer.py https://huggingface.co/spaces/Ai-Bharti/dataSraping/resolve/main/data_normalizer.py
19.6 kB
| """ | |
| data_normalizer.py - Smart Data Extraction + Normalization | |
| =========================================================== | |
| Yeh module kya karta hai: | |
| β USD β INR Cr convert karta hai | |
| β Million/Billion/Lakh β Cr convert karta hai | |
| β Timeline ko standard format mein laata hai | |
| β Customer, Order Type, Date accurately nikalta hai | |
| β SEBI Annexure-A table structure samajhta hai (jo aapke PDFs mein hai) | |
| Real-world examples jo handle karta hai: | |
| USD 1,346,210 β 1.14 Cr (@ Rs.85/USD) | |
| βΉ1,587.10 million β 158.71 Cr | |
| Rs. 890.69 Million β 89.07 Cr | |
| Rs. 17,750 Million β 1775.00 Cr | |
| INR 20.58 Crores β 20.58 Cr | |
| Rs. 1,827.33 β 1827.33 Cr (already in Cr) | |
| Delivery by Oct 2027 β Oct 2027 | |
| Within 8 months β 8 Months | |
| 910 Days β 910 Days (~30 Months) | |
| 4 (04) Years β 4 Years (~48 Months) | |
| 18 Months β 18 Months | |
| """ | |
| import re | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # π± CURRENCY CONVERSION RATES | |
| # Inhe update karte rahna agar rate badal jaye | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| USD_TO_INR = 85.0 # 1 USD = Rs. 85 approx | |
| MILLION_TO_CR = 0.1 # 1 Million = 0.1 Cr (10 Lakh) | |
| BILLION_TO_CR = 100.0 # 1 Billion = 100 Cr | |
| LAKH_TO_CR = 0.01 # 1 Lakh = 0.01 Cr | |
| MN_TO_CR = 0.1 # MN = Million shorthand | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # π° MAIN FUNCTION: Order Value Normalize karo | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def normalize_order_value(text: str) -> dict: | |
| """ | |
| Kisi bhi format ki value ko INR Crore mein convert karta hai. | |
| Input examples: | |
| "USD 1,346,210" | |
| "βΉ1,587.10 million (Including GST)" | |
| "Rs. 890.69 Million" | |
| "Rs. 1,827.33 Cr" | |
| "INR 20.58 Crores" | |
| "Rs. 17,750 Million" | |
| Return: | |
| { | |
| "original": "USD 1,346,210", | |
| "inr_crore": 1.14, | |
| "inr_crore_display": "1.14 Cr", | |
| "currency": "USD", | |
| "note": "Converted @ Rs.85/USD" | |
| } | |
| """ | |
| if not text or text.strip() in ["N/A", "", "-", "β"]: | |
| return _empty_value() | |
| clean = text.strip() | |
| # ββ Pattern 1: USD amount ββ | |
| # Matches: "USD 1,346,210" or "USD 1346210" or "US$ 1,346,210" | |
| usd_match = re.search( | |
| r"USD\s*[\$]?\s*([\d,]+(?:\.\d+)?)", | |
| clean, re.IGNORECASE | |
| ) | |
| if usd_match: | |
| usd_val = float(usd_match.group(1).replace(",", "")) | |
| inr_val = usd_val * USD_TO_INR | |
| inr_cr = round(inr_val / 1e7, 2) # Convert to Crore | |
| return { | |
| "original": clean, | |
| "inr_crore": inr_cr, | |
| "inr_crore_display": f"{inr_cr:.2f} Cr", | |
| "currency": "USD", | |
| "note": f"USD {usd_val:,.0f} Γ Rs.{USD_TO_INR}/USD Γ· 1Cr" | |
| } | |
| # ββ Pattern 2: Rupees in Million ββ | |
| # Matches: "Rs. 890.69 Million", "βΉ1,587.10 million", "INR 17,750 Million", "Rs. 17,750 Mn" | |
| million_match = re.search( | |
| r"(?:Rs\.?|βΉ|INR|INR\s*)?\s*([\d,]+(?:\.\d+)?)\s*(?:million|mn\.?)\b", | |
| clean, re.IGNORECASE | |
| ) | |
| if million_match: | |
| mn_val = float(million_match.group(1).replace(",", "")) | |
| cr_val = round(mn_val * MILLION_TO_CR, 2) | |
| return { | |
| "original": clean, | |
| "inr_crore": cr_val, | |
| "inr_crore_display": f"{cr_val:.2f} Cr", | |
| "currency": "INR", | |
| "note": f"Rs.{mn_val} Mn Γ· 10 = {cr_val} Cr" | |
| } | |
| # ββ Pattern 3: Rupees in Billion ββ | |
| # Matches: "Rs. 2.5 Billion", "INR 1 Billion" | |
| billion_match = re.search( | |
| r"(?:Rs\.?|βΉ|INR)?\s*([\d,]+(?:\.\d+)?)\s*(?:billion|bn\.?)\b", | |
| clean, re.IGNORECASE | |
| ) | |
| if billion_match: | |
| bn_val = float(billion_match.group(1).replace(",", "")) | |
| cr_val = round(bn_val * BILLION_TO_CR, 2) | |
| return { | |
| "original": clean, | |
| "inr_crore": cr_val, | |
| "inr_crore_display": f"{cr_val:.2f} Cr", | |
| "currency": "INR", | |
| "note": f"Rs.{bn_val} Bn Γ 100 = {cr_val} Cr" | |
| } | |
| # ββ Pattern 4: Already in Crore ββ | |
| # Matches: "Rs. 1,827.33 Cr", "INR 20.58 Crores", "Rs. 775.27 Cr. approx" | |
| # Also: "Rs 148 crores", "approx. Rs. 33 Crore" | |
| crore_match = re.search( | |
| r"(?:Rs\.?|βΉ|INR)?\s*([\d,]+(?:\.\d+)?)\s*(?:crore|cr\.?|crores)\b", | |
| clean, re.IGNORECASE | |
| ) | |
| if crore_match: | |
| cr_val = round(float(crore_match.group(1).replace(",", "")), 2) | |
| return { | |
| "original": clean, | |
| "inr_crore": cr_val, | |
| "inr_crore_display": f"{cr_val:.2f} Cr", | |
| "currency": "INR", | |
| "note": "Already in Crore" | |
| } | |
| # ββ Pattern 5: Rupees in Lakh ββ | |
| # Matches: "Rs. 50 Lakh", "Rs. 2.5 L" | |
| lakh_match = re.search( | |
| r"(?:Rs\.?|βΉ|INR)?\s*([\d,]+(?:\.\d+)?)\s*(?:lakh|lac|l)\b", | |
| clean, re.IGNORECASE | |
| ) | |
| if lakh_match: | |
| lakh_val = float(lakh_match.group(1).replace(",", "")) | |
| cr_val = round(lakh_val * LAKH_TO_CR, 4) | |
| return { | |
| "original": clean, | |
| "inr_crore": cr_val, | |
| "inr_crore_display": f"{cr_val:.4f} Cr", | |
| "currency": "INR", | |
| "note": f"Rs.{lakh_val} L Γ· 100 = {cr_val} Cr" | |
| } | |
| # ββ Fallback: Kuch match nahi hua ββ | |
| return { | |
| "original": clean, | |
| "inr_crore": None, | |
| "inr_crore_display": clean, # As-is rakh do | |
| "currency": "UNKNOWN", | |
| "note": "Auto-convert nahi hua, manual check karo" | |
| } | |
| def _empty_value(): | |
| return { | |
| "original": "", "inr_crore": None, | |
| "inr_crore_display": "N/A", "currency": "", "note": "" | |
| } | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # β° TIMELINE NORMALIZE karo | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def normalize_timeline(text: str) -> dict: | |
| """ | |
| Kisi bhi format ki timeline ko standard format mein convert karta hai. | |
| Input examples: | |
| "910 Days" β {"display": "910 Days (~30 Months)", "months": 30} | |
| "4 (04) Years" β {"display": "4 Years (~48 Months)", "months": 48} | |
| "Within 18 Months" β {"display": "18 Months", "months": 18} | |
| "Delivery to be completed by Aug 31, 2026" β {"display": "Aug 2026", "months": None} | |
| "Delivery by October 2027" β {"display": "Oct 2027", "months": None} | |
| "Within 8 months from LOA" β {"display": "8 Months from LOA", "months": 8} | |
| "11 months from Letter of Acceptance" β same pattern | |
| """ | |
| if not text or text.strip() in ["N/A", "", "-"]: | |
| return {"display": "N/A", "months": None, "original": text} | |
| clean = text.strip() | |
| months = None | |
| display = clean # Default: as-is | |
| # ββ Pattern 1: Days ββ | |
| # "910 Days", "30 days" | |
| days_match = re.search(r"(\d+)\s*days?\b", clean, re.IGNORECASE) | |
| if days_match: | |
| days = int(days_match.group(1)) | |
| months = round(days / 30.4) | |
| display = f"{days} Days (~{months} Months)" | |
| return {"display": display, "months": months, "original": clean} | |
| # ββ Pattern 2: Years ββ | |
| # "4 Years", "4 (04) Years", "Four (04) Years", "2.5 years" | |
| years_match = re.search( | |
| r"(\d+(?:\.\d+)?)\s*(?:\(\d+\)\s*)?years?\b", | |
| clean, re.IGNORECASE | |
| ) | |
| if years_match: | |
| years = float(years_match.group(1)) | |
| months = round(years * 12) | |
| display = f"{int(years) if years == int(years) else years} Years (~{months} Months)" | |
| return {"display": display, "months": months, "original": clean} | |
| # ββ Pattern 3: Months ββ | |
| # "18 Months", "Within 18 Months", "8 months from LOA" | |
| months_match = re.search(r"(\d+)\s*months?\b", clean, re.IGNORECASE) | |
| if months_match: | |
| months = int(months_match.group(1)) | |
| # Context nikalo - "from LOA" jaisi cheez | |
| context_match = re.search( | |
| r"months?\s+(from\s+[^\.,]{3,40})", clean, re.IGNORECASE | |
| ) | |
| suffix = f" {context_match.group(1)}" if context_match else "" | |
| display = f"{months} Months{suffix}" | |
| return {"display": display, "months": months, "original": clean} | |
| # ββ Pattern 4: Specific Date ββ | |
| # "Delivery by October 2027", "by August 31, 2026", "by April 2026" | |
| date_match = re.search( | |
| r"(?:by|until|before|on)\s+" | |
| r"((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*" | |
| r"(?:\s+\d{1,2},?)?\s+\d{4})", | |
| clean, re.IGNORECASE | |
| ) | |
| if date_match: | |
| display = date_match.group(1).strip() | |
| # Short month format: "October 2027" β "Oct 2027" | |
| display = _shorten_month(display) | |
| return {"display": display, "months": None, "original": clean} | |
| # ββ Fallback ββ | |
| return {"display": clean[:60], "months": None, "original": clean} | |
| def _shorten_month(date_str: str) -> str: | |
| """'October 2027' β 'Oct 2027'""" | |
| month_map = { | |
| "january": "Jan", "february": "Feb", "march": "Mar", | |
| "april": "Apr", "may": "May", "june": "Jun", | |
| "july": "Jul", "august": "Aug", "september": "Sep", | |
| "october": "Oct", "november": "Nov", "december": "Dec" | |
| } | |
| for full, short in month_map.items(): | |
| date_str = re.sub(full, short, date_str, flags=re.IGNORECASE) | |
| return date_str | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # π SEBI ANNEXURE-A TABLE PARSER | |
| # Yeh special function hai jo BSE filings ki standard table padh sakta hai | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def parse_sebi_annexure_table(text: str) -> dict: | |
| """ | |
| SEBI Regulation 30 ke standard Annexure-A table se data nikalta hai. | |
| Yeh table structure handle karta hai: | |
| Row 1: Name of entity awarding β Customer | |
| Row 2: Significant terms β Order Description | |
| Row 4: Nature of order β Order Type | |
| Row 6: Time period β Timeline | |
| Row 7: Broad consideration / size β Order Value | |
| Yeh almost SABB BSE filings mein hota hai! | |
| """ | |
| result = { | |
| "customer": "", | |
| "order_description": "", | |
| "order_type": "", | |
| "timeline_raw": "", | |
| "value_raw": "", | |
| "domestic_or_intl": "", | |
| } | |
| if not text: | |
| return result | |
| # Normalize whitespace | |
| text_clean = re.sub(r"\s+", " ", text) | |
| # ββ Row 1: Customer/Entity Name ββ | |
| cust_patterns = [ | |
| r"(?:name of the entity awarding[^:]*?[:]\s*)([^\n\r]{5,100}?)(?:\n|\r|$|\d+\.)", | |
| r"(?:entity awarding[^:]*?[:]\s*)([^\n\r]{5,100}?)(?:\n|\r|$)", | |
| r"(?:awarded by|awarded from)[:\s]+([A-Z][^\n\r]{5,100}?)(?:\n|\r|$)", | |
| ] | |
| for p in cust_patterns: | |
| m = re.search(p, text_clean, re.IGNORECASE) | |
| if m: | |
| val = m.group(1).strip() | |
| # Tabular "Details" header skip karo | |
| if len(val) > 3 and "particular" not in val.lower(): | |
| result["customer"] = val[:120] | |
| break | |
| # ββ Row 2: Order Description / Terms ββ | |
| desc_patterns = [ | |
| r"(?:significant terms[^:]*?[:]\s*)([^\n\r]{10,300}?)(?:\n|\r|$|\d+\.)", | |
| r"(?:terms and conditions[^:]*?[:]\s*)([^\n\r]{10,300}?)(?:\n|\r|$)", | |
| ] | |
| for p in desc_patterns: | |
| m = re.search(p, text_clean, re.IGNORECASE) | |
| if m: | |
| result["order_description"] = m.group(1).strip()[:300] | |
| break | |
| # ββ Row 4: Nature of Order ββ | |
| nature_patterns = [ | |
| r"(?:nature of order[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$|\d+\.)", | |
| ] | |
| for p in nature_patterns: | |
| m = re.search(p, text_clean, re.IGNORECASE) | |
| if m: | |
| result["order_type"] = m.group(1).strip()[:150] | |
| break | |
| # ββ Domestic/International ββ | |
| dom_match = re.search( | |
| r"(?:domestic or international[^:]*?[:]\s*)([^\n\r]{3,30}?)(?:\n|\r|$|\d+\.)", | |
| text_clean, re.IGNORECASE | |
| ) | |
| if dom_match: | |
| result["domestic_or_intl"] = dom_match.group(1).strip() | |
| # ββ Row 6: Timeline ββ | |
| time_patterns = [ | |
| r"(?:time period[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$|\d+\.)", | |
| r"(?:period by which[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$)", | |
| r"(?:to be executed[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$)", | |
| ] | |
| for p in time_patterns: | |
| m = re.search(p, text_clean, re.IGNORECASE) | |
| if m: | |
| result["timeline_raw"] = m.group(1).strip()[:200] | |
| break | |
| # ββ Row 7: Order Value ββ | |
| value_patterns = [ | |
| r"(?:broad consideration[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$|\d+\.)", | |
| r"(?:size of the order[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$)", | |
| r"(?:consideration or size[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$)", | |
| r"(?:value of order[^:]*?[:]\s*)([^\n\r]{5,200}?)(?:\n|\r|$)", | |
| ] | |
| for p in value_patterns: | |
| m = re.search(p, text_clean, re.IGNORECASE) | |
| if m: | |
| result["value_raw"] = m.group(1).strip()[:200] | |
| break | |
| return result | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # π― MASTER FUNCTION - Sab ek jagah | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def extract_and_normalize(pdf_text: str, snippet: str, filing_meta: dict) -> dict: | |
| """ | |
| PDF text se FULL normalized record banata hai. | |
| Pehle SEBI Annexure table parse karta hai (most reliable), | |
| phir value + timeline normalize karta hai. | |
| Return: Complete record dict for CSV | |
| """ | |
| # Step 1: SEBI standard table parse karo | |
| annexure = parse_sebi_annexure_table(pdf_text) | |
| # Step 2: Value normalize karo | |
| raw_value = annexure["value_raw"] or snippet | |
| value_info = normalize_order_value(raw_value) | |
| # Step 3: Timeline normalize karo | |
| raw_timeline = annexure["timeline_raw"] or "" | |
| timeline_info = normalize_timeline(raw_timeline) | |
| # Step 4: Customer finalize | |
| customer = annexure["customer"] | |
| if not customer: | |
| # Fallback: snippet se try karo | |
| m = re.search( | |
| r"(?:from|by)\s+([A-Z][A-Za-z\s&\-]{5,60}" | |
| r"(?:Ltd\.?|Limited|Authority|Board|NHAI|DDA|NTPC|ONGC|RAIL|NAVY|ARMY))", | |
| snippet | |
| ) | |
| if m: | |
| customer = m.group(1).strip() | |
| # Step 5: Order type finalize | |
| order_type = annexure["order_type"] or filing_meta.get("title", "") | |
| # Title se Order type nikalo agar table mein nahi mila | |
| if "LOA" in order_type or "Letter of Award" in (pdf_text + snippet): | |
| order_type = "Letter of Award" | |
| elif "LOI" in order_type or "Letter of Intent" in (pdf_text + snippet): | |
| order_type = "Letter of Intent" | |
| elif "Work Order" in (pdf_text + snippet): | |
| order_type = "Work Order" | |
| return { | |
| "Stock Name": filing_meta.get("stock", ""), | |
| "Exchange": filing_meta.get("exchange", ""), | |
| "Filing Title": filing_meta.get("title", ""), | |
| "Customer": customer or "N/A", | |
| "Order Description": annexure["order_description"] or "N/A", | |
| "Order Type": order_type or "N/A", | |
| "Domestic / International": annexure["domestic_or_intl"] or "N/A", | |
| "Date Order Received": filing_meta.get("date", ""), | |
| # β NORMALIZED VALUES | |
| "Order Value (INR Cr)": value_info["inr_crore_display"], | |
| "Value (Raw)": value_info["original"], | |
| "Conversion Note": value_info["note"], | |
| # β NORMALIZED TIMELINE | |
| "Timeline": timeline_info["display"], | |
| "Timeline (Months)": str(timeline_info["months"]) if timeline_info["months"] else "N/A", | |
| "Timeline (Raw)": timeline_info["original"], | |
| "Source PDF": filing_meta.get("pdf_url", ""), | |
| "Screener Date": filing_meta.get("date", ""), | |
| "Scraped At": __import__("datetime").datetime.now().strftime("%Y-%m-%d %H:%M"), | |
| } | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # π§ͺ TEST - Seedha run karo to verify | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| if __name__ == "__main__": | |
| print("=" * 55) | |
| print("π° ORDER VALUE CONVERSION TESTS") | |
| print("=" * 55) | |
| test_values = [ | |
| "USD 1,346,210", | |
| "USD 1346210.00", | |
| "βΉ1,587.10 million (Including GST)", | |
| "Rs. 890.69 Million (including GST)", | |
| "Rs. 17,750 Million", | |
| "INR 20.58 Crores (Including Tax)", | |
| "Rs. 1,827.33 Cr", | |
| "Rs. 148 crores", | |
| "Rs. 775.27 Cr. approx.", | |
| "Rs. 33 Crore (Excl. GST)", | |
| "Rs. 2.13 crores (excluding taxes)", | |
| "Rs. 437.79 Cr. (Approx.)", | |
| "Rs. 1,582.11", # Pure number - Cr assume karein | |
| ] | |
| for val in test_values: | |
| result = normalize_order_value(val) | |
| print(f" Input : {val}") | |
| print(f" Output: {result['inr_crore_display']} [{result['note']}]") | |
| print() | |
| print("=" * 55) | |
| print("β° TIMELINE NORMALIZATION TESTS") | |
| print("=" * 55) | |
| test_timelines = [ | |
| "910 Days", | |
| "Four (04) Years", | |
| "4 (04) Years", | |
| "Within 18 Months from signing of PPA", | |
| "Delivery to be completed by August 31, 2026", | |
| "Delivery by October 2027", | |
| "Within 8 months from the Letter of Award", | |
| "11 months from the date of issue of Letter of Acceptance", | |
| "Delivery to be completed by April 2026", | |
| "25 years from commissioning", | |
| ] | |
| for tl in test_timelines: | |
| result = normalize_timeline(tl) | |
| months_str = f" | {result['months']} months" if result["months"] else "" | |
| print(f" Input : {tl}") | |
| print(f" Output: {result['display']}{months_str}") | |
| print() | |