Spaces:
Build error
Build error
Commit Β·
563cd5e
1
Parent(s): f0cecec
feat(scrapers): add CAG audit report scraper
Browse files- Scrapes Comptroller & Auditor General reports from cag.gov.in
- Detects fraud keywords: misappropriation, ghost beneficiaries, etc.
- get_high_value_irregularities() filters reports by crore amount
- Sample data based on real CAG findings (TN MGNREGA, Assam PM-KISAN)
- Key for graph: links audit findings to schemes, states, departments
- 3s polite delay, falls back to structured sample data
- scrapers/cag_scraper.py +233 -0
scrapers/cag_scraper.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
BharatGraph - CAG (Comptroller and Auditor General) Scraper
|
| 3 |
+
Fetches audit reports and findings from India's supreme audit institution.
|
| 4 |
+
Source: cag.gov.in (official government site, 100% public)
|
| 5 |
+
|
| 6 |
+
WHY THIS MATTERS:
|
| 7 |
+
CAG audits expose exactly the type of fraud we want to map:
|
| 8 |
+
- Funds released but works not done
|
| 9 |
+
- Ghost beneficiaries in schemes
|
| 10 |
+
- Irregular contract awards
|
| 11 |
+
- Disproportionate spending
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import re
|
| 15 |
+
import json
|
| 16 |
+
from datetime import datetime
|
| 17 |
+
from bs4 import BeautifulSoup
|
| 18 |
+
from scrapers.base_scraper import BaseScraper
|
| 19 |
+
from loguru import logger
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class CAGScraper(BaseScraper):
|
| 23 |
+
"""
|
| 24 |
+
Scrapes CAG audit reports from cag.gov.in.
|
| 25 |
+
|
| 26 |
+
CAG (Comptroller and Auditor General of India) is the
|
| 27 |
+
constitutional authority that audits all government accounts.
|
| 28 |
+
Their reports are gold for finding financial irregularities.
|
| 29 |
+
|
| 30 |
+
Data includes:
|
| 31 |
+
- Performance audit reports (scheme-wise)
|
| 32 |
+
- State finance audit reports
|
| 33 |
+
- Union government audit reports
|
| 34 |
+
- Amount of irregularities found (in crores)
|
| 35 |
+
- Departments/ministries flagged
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
BASE_URL = "https://cag.gov.in"
|
| 39 |
+
REPORTS_URL = "https://cag.gov.in/en/audit-report"
|
| 40 |
+
|
| 41 |
+
# Keywords that indicate financial irregularities
|
| 42 |
+
FRAUD_KEYWORDS = [
|
| 43 |
+
"misappropriation", "irregularities", "excess payment",
|
| 44 |
+
"short recovery", "loss", "avoidable expenditure",
|
| 45 |
+
"fictitious", "ghost", "diversion of funds", "unspent",
|
| 46 |
+
"non-utilization", "fraudulent", "collusion", "overpayment",
|
| 47 |
+
"undue benefit", "unauthorized", "infructuous",
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
def __init__(self):
|
| 51 |
+
super().__init__(name="cag", delay=3.0)
|
| 52 |
+
|
| 53 |
+
def fetch_report_list(self, limit: int = 20) -> list:
|
| 54 |
+
"""
|
| 55 |
+
Fetch list of recent CAG audit reports.
|
| 56 |
+
Returns list of report metadata dicts.
|
| 57 |
+
"""
|
| 58 |
+
logger.info("[CAG] Fetching audit report list...")
|
| 59 |
+
html = self.get_html(self.REPORTS_URL)
|
| 60 |
+
|
| 61 |
+
if not html:
|
| 62 |
+
logger.warning("[CAG] Could not fetch report list, using sample")
|
| 63 |
+
return self._get_sample_reports()
|
| 64 |
+
|
| 65 |
+
return self._parse_report_list(html, limit)
|
| 66 |
+
|
| 67 |
+
def _parse_report_list(self, html: str, limit: int) -> list:
|
| 68 |
+
"""Parse report links and metadata from CAG website."""
|
| 69 |
+
soup = BeautifulSoup(html, "lxml")
|
| 70 |
+
reports = []
|
| 71 |
+
|
| 72 |
+
# CAG site uses various link patterns for reports
|
| 73 |
+
links = soup.find_all("a", href=True)
|
| 74 |
+
|
| 75 |
+
for link in links:
|
| 76 |
+
href = link.get("href", "")
|
| 77 |
+
text = link.get_text(strip=True)
|
| 78 |
+
|
| 79 |
+
# Filter for actual report links
|
| 80 |
+
if not text or len(text) < 10:
|
| 81 |
+
continue
|
| 82 |
+
if not any(kw in href.lower() for kw in
|
| 83 |
+
["report", "audit", "pdf", "ar-"]):
|
| 84 |
+
continue
|
| 85 |
+
|
| 86 |
+
# Build full URL
|
| 87 |
+
if href.startswith("http"):
|
| 88 |
+
full_url = href
|
| 89 |
+
elif href.startswith("/"):
|
| 90 |
+
full_url = self.BASE_URL + href
|
| 91 |
+
else:
|
| 92 |
+
continue
|
| 93 |
+
|
| 94 |
+
report = {
|
| 95 |
+
"title": text,
|
| 96 |
+
"url": full_url,
|
| 97 |
+
"source": "CAG India",
|
| 98 |
+
"source_url": self.BASE_URL,
|
| 99 |
+
"scraped_at": datetime.now().isoformat(),
|
| 100 |
+
"entity_type": "audit_report",
|
| 101 |
+
# Flag if title mentions irregularities
|
| 102 |
+
"alert_keywords": self._find_fraud_keywords(text),
|
| 103 |
+
}
|
| 104 |
+
reports.append(report)
|
| 105 |
+
|
| 106 |
+
if len(reports) >= limit:
|
| 107 |
+
break
|
| 108 |
+
|
| 109 |
+
logger.info(f"[CAG] Parsed {len(reports)} report links")
|
| 110 |
+
return reports if reports else self._get_sample_reports()
|
| 111 |
+
|
| 112 |
+
def _find_fraud_keywords(self, text: str) -> list:
|
| 113 |
+
"""Find fraud-related keywords in report title."""
|
| 114 |
+
text_lower = text.lower()
|
| 115 |
+
return [kw for kw in self.FRAUD_KEYWORDS if kw in text_lower]
|
| 116 |
+
|
| 117 |
+
def _get_sample_reports(self) -> list:
|
| 118 |
+
"""
|
| 119 |
+
Sample CAG report structures for testing.
|
| 120 |
+
Based on real CAG report formats.
|
| 121 |
+
"""
|
| 122 |
+
return [
|
| 123 |
+
{
|
| 124 |
+
"title":
|
| 125 |
+
"Report on MGNREGS - Irregularities in Tamil Nadu 2023",
|
| 126 |
+
"url":
|
| 127 |
+
"https://cag.gov.in/en/audit-report/details/sample-1",
|
| 128 |
+
"year": "2023",
|
| 129 |
+
"state": "Tamil Nadu",
|
| 130 |
+
"scheme": "MGNREGS",
|
| 131 |
+
"amount_crore": 112.81,
|
| 132 |
+
"irregularity_type": "Misappropriation of funds",
|
| 133 |
+
"finding":
|
| 134 |
+
"38,063 irregularities found. Works not executed "
|
| 135 |
+
"despite funds released.",
|
| 136 |
+
"source": "CAG India (sample)",
|
| 137 |
+
"scraped_at": datetime.now().isoformat(),
|
| 138 |
+
"entity_type": "audit_report",
|
| 139 |
+
"alert_keywords": ["irregularities", "misappropriation"],
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"title":
|
| 143 |
+
"Performance Audit: PM-KISAN Scheme - Assam 2024",
|
| 144 |
+
"url":
|
| 145 |
+
"https://cag.gov.in/en/audit-report/details/sample-2",
|
| 146 |
+
"year": "2024",
|
| 147 |
+
"state": "Assam",
|
| 148 |
+
"scheme": "PM-KISAN",
|
| 149 |
+
"amount_crore": 768.3,
|
| 150 |
+
"irregularity_type": "Ineligible beneficiaries",
|
| 151 |
+
"finding":
|
| 152 |
+
"35% applicants ineligible. 1.56 million bogus entries "
|
| 153 |
+
"including govt employees and pensioners.",
|
| 154 |
+
"source": "CAG India (sample)",
|
| 155 |
+
"scraped_at": datetime.now().isoformat(),
|
| 156 |
+
"entity_type": "audit_report",
|
| 157 |
+
"alert_keywords": ["irregularities", "fictitious"],
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"title":
|
| 161 |
+
"Audit Report: Public Procurement Irregularities 2023",
|
| 162 |
+
"url":
|
| 163 |
+
"https://cag.gov.in/en/audit-report/details/sample-3",
|
| 164 |
+
"year": "2023",
|
| 165 |
+
"state": "National",
|
| 166 |
+
"scheme": "Public Procurement",
|
| 167 |
+
"amount_crore": 450.0,
|
| 168 |
+
"irregularity_type": "Bid rigging and collusion",
|
| 169 |
+
"finding":
|
| 170 |
+
"Single-bid contracts awarded without competition. "
|
| 171 |
+
"Repeated winners across multiple tenders.",
|
| 172 |
+
"source": "CAG India (sample)",
|
| 173 |
+
"scraped_at": datetime.now().isoformat(),
|
| 174 |
+
"entity_type": "audit_report",
|
| 175 |
+
"alert_keywords": ["collusion", "irregularities"],
|
| 176 |
+
},
|
| 177 |
+
]
|
| 178 |
+
|
| 179 |
+
def fetch_and_save(self, save: bool = True) -> list:
|
| 180 |
+
"""Fetch all reports and save to disk."""
|
| 181 |
+
reports = self.fetch_report_list(limit=30)
|
| 182 |
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 183 |
+
|
| 184 |
+
if save and reports:
|
| 185 |
+
filepath = f"data/samples/cag_reports_{timestamp}.json"
|
| 186 |
+
self.save_json(reports, filepath)
|
| 187 |
+
logger.success(f"[CAG] Saved {len(reports)} reports to {filepath}")
|
| 188 |
+
|
| 189 |
+
return reports
|
| 190 |
+
|
| 191 |
+
def get_high_value_irregularities(self, reports: list,
|
| 192 |
+
min_crore: float = 10.0) -> list:
|
| 193 |
+
"""
|
| 194 |
+
Filter reports where amount involved >= min_crore.
|
| 195 |
+
Helps prioritize large-scale fraud for investigation.
|
| 196 |
+
"""
|
| 197 |
+
flagged = [
|
| 198 |
+
r for r in reports
|
| 199 |
+
if float(r.get("amount_crore", 0) or 0) >= min_crore
|
| 200 |
+
]
|
| 201 |
+
logger.info(f"[CAG] Reports >= βΉ{min_crore}Cr: {len(flagged)}")
|
| 202 |
+
return flagged
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
# ββ Run directly to test ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 206 |
+
if __name__ == "__main__":
|
| 207 |
+
print("=" * 60)
|
| 208 |
+
print("BharatGraph - CAG Audit Report Scraper Test")
|
| 209 |
+
print("=" * 60)
|
| 210 |
+
|
| 211 |
+
scraper = CAGScraper()
|
| 212 |
+
|
| 213 |
+
print("\n[1] Fetching CAG audit report list...")
|
| 214 |
+
reports = scraper.fetch_report_list(limit=20)
|
| 215 |
+
print(f" Total reports found: {len(reports)}")
|
| 216 |
+
|
| 217 |
+
if reports:
|
| 218 |
+
r = reports[0]
|
| 219 |
+
print(f"\n Example report:")
|
| 220 |
+
print(f" Title: {r.get('title', 'N/A')[:60]}")
|
| 221 |
+
print(f" URL: {r.get('url', 'N/A')[:60]}")
|
| 222 |
+
print(f" Alert kws: {r.get('alert_keywords', [])}")
|
| 223 |
+
if r.get("amount_crore"):
|
| 224 |
+
print(f" Amount: βΉ{r.get('amount_crore')} Crore")
|
| 225 |
+
|
| 226 |
+
print("\n[2] Filtering high-value irregularities (>=10 Cr)...")
|
| 227 |
+
big = scraper.get_high_value_irregularities(reports, min_crore=10.0)
|
| 228 |
+
print(f" High-value reports: {len(big)}")
|
| 229 |
+
|
| 230 |
+
print("\n[3] Saving all reports...")
|
| 231 |
+
all_reports = scraper.fetch_and_save(save=True)
|
| 232 |
+
print(f" Saved {len(all_reports)} reports to data/samples/")
|
| 233 |
+
print("\nDone!")
|