abinazebinoy commited on
Commit
563cd5e
Β·
1 Parent(s): f0cecec

feat(scrapers): add CAG audit report scraper

Browse files

- Scrapes Comptroller & Auditor General reports from cag.gov.in
- Detects fraud keywords: misappropriation, ghost beneficiaries, etc.
- get_high_value_irregularities() filters reports by crore amount
- Sample data based on real CAG findings (TN MGNREGA, Assam PM-KISAN)
- Key for graph: links audit findings to schemes, states, departments
- 3s polite delay, falls back to structured sample data

Files changed (1) hide show
  1. scrapers/cag_scraper.py +233 -0
scrapers/cag_scraper.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ BharatGraph - CAG (Comptroller and Auditor General) Scraper
3
+ Fetches audit reports and findings from India's supreme audit institution.
4
+ Source: cag.gov.in (official government site, 100% public)
5
+
6
+ WHY THIS MATTERS:
7
+ CAG audits expose exactly the type of fraud we want to map:
8
+ - Funds released but works not done
9
+ - Ghost beneficiaries in schemes
10
+ - Irregular contract awards
11
+ - Disproportionate spending
12
+ """
13
+
14
+ import re
15
+ import json
16
+ from datetime import datetime
17
+ from bs4 import BeautifulSoup
18
+ from scrapers.base_scraper import BaseScraper
19
+ from loguru import logger
20
+
21
+
22
+ class CAGScraper(BaseScraper):
23
+ """
24
+ Scrapes CAG audit reports from cag.gov.in.
25
+
26
+ CAG (Comptroller and Auditor General of India) is the
27
+ constitutional authority that audits all government accounts.
28
+ Their reports are gold for finding financial irregularities.
29
+
30
+ Data includes:
31
+ - Performance audit reports (scheme-wise)
32
+ - State finance audit reports
33
+ - Union government audit reports
34
+ - Amount of irregularities found (in crores)
35
+ - Departments/ministries flagged
36
+ """
37
+
38
+ BASE_URL = "https://cag.gov.in"
39
+ REPORTS_URL = "https://cag.gov.in/en/audit-report"
40
+
41
+ # Keywords that indicate financial irregularities
42
+ FRAUD_KEYWORDS = [
43
+ "misappropriation", "irregularities", "excess payment",
44
+ "short recovery", "loss", "avoidable expenditure",
45
+ "fictitious", "ghost", "diversion of funds", "unspent",
46
+ "non-utilization", "fraudulent", "collusion", "overpayment",
47
+ "undue benefit", "unauthorized", "infructuous",
48
+ ]
49
+
50
+ def __init__(self):
51
+ super().__init__(name="cag", delay=3.0)
52
+
53
+ def fetch_report_list(self, limit: int = 20) -> list:
54
+ """
55
+ Fetch list of recent CAG audit reports.
56
+ Returns list of report metadata dicts.
57
+ """
58
+ logger.info("[CAG] Fetching audit report list...")
59
+ html = self.get_html(self.REPORTS_URL)
60
+
61
+ if not html:
62
+ logger.warning("[CAG] Could not fetch report list, using sample")
63
+ return self._get_sample_reports()
64
+
65
+ return self._parse_report_list(html, limit)
66
+
67
+ def _parse_report_list(self, html: str, limit: int) -> list:
68
+ """Parse report links and metadata from CAG website."""
69
+ soup = BeautifulSoup(html, "lxml")
70
+ reports = []
71
+
72
+ # CAG site uses various link patterns for reports
73
+ links = soup.find_all("a", href=True)
74
+
75
+ for link in links:
76
+ href = link.get("href", "")
77
+ text = link.get_text(strip=True)
78
+
79
+ # Filter for actual report links
80
+ if not text or len(text) < 10:
81
+ continue
82
+ if not any(kw in href.lower() for kw in
83
+ ["report", "audit", "pdf", "ar-"]):
84
+ continue
85
+
86
+ # Build full URL
87
+ if href.startswith("http"):
88
+ full_url = href
89
+ elif href.startswith("/"):
90
+ full_url = self.BASE_URL + href
91
+ else:
92
+ continue
93
+
94
+ report = {
95
+ "title": text,
96
+ "url": full_url,
97
+ "source": "CAG India",
98
+ "source_url": self.BASE_URL,
99
+ "scraped_at": datetime.now().isoformat(),
100
+ "entity_type": "audit_report",
101
+ # Flag if title mentions irregularities
102
+ "alert_keywords": self._find_fraud_keywords(text),
103
+ }
104
+ reports.append(report)
105
+
106
+ if len(reports) >= limit:
107
+ break
108
+
109
+ logger.info(f"[CAG] Parsed {len(reports)} report links")
110
+ return reports if reports else self._get_sample_reports()
111
+
112
+ def _find_fraud_keywords(self, text: str) -> list:
113
+ """Find fraud-related keywords in report title."""
114
+ text_lower = text.lower()
115
+ return [kw for kw in self.FRAUD_KEYWORDS if kw in text_lower]
116
+
117
+ def _get_sample_reports(self) -> list:
118
+ """
119
+ Sample CAG report structures for testing.
120
+ Based on real CAG report formats.
121
+ """
122
+ return [
123
+ {
124
+ "title":
125
+ "Report on MGNREGS - Irregularities in Tamil Nadu 2023",
126
+ "url":
127
+ "https://cag.gov.in/en/audit-report/details/sample-1",
128
+ "year": "2023",
129
+ "state": "Tamil Nadu",
130
+ "scheme": "MGNREGS",
131
+ "amount_crore": 112.81,
132
+ "irregularity_type": "Misappropriation of funds",
133
+ "finding":
134
+ "38,063 irregularities found. Works not executed "
135
+ "despite funds released.",
136
+ "source": "CAG India (sample)",
137
+ "scraped_at": datetime.now().isoformat(),
138
+ "entity_type": "audit_report",
139
+ "alert_keywords": ["irregularities", "misappropriation"],
140
+ },
141
+ {
142
+ "title":
143
+ "Performance Audit: PM-KISAN Scheme - Assam 2024",
144
+ "url":
145
+ "https://cag.gov.in/en/audit-report/details/sample-2",
146
+ "year": "2024",
147
+ "state": "Assam",
148
+ "scheme": "PM-KISAN",
149
+ "amount_crore": 768.3,
150
+ "irregularity_type": "Ineligible beneficiaries",
151
+ "finding":
152
+ "35% applicants ineligible. 1.56 million bogus entries "
153
+ "including govt employees and pensioners.",
154
+ "source": "CAG India (sample)",
155
+ "scraped_at": datetime.now().isoformat(),
156
+ "entity_type": "audit_report",
157
+ "alert_keywords": ["irregularities", "fictitious"],
158
+ },
159
+ {
160
+ "title":
161
+ "Audit Report: Public Procurement Irregularities 2023",
162
+ "url":
163
+ "https://cag.gov.in/en/audit-report/details/sample-3",
164
+ "year": "2023",
165
+ "state": "National",
166
+ "scheme": "Public Procurement",
167
+ "amount_crore": 450.0,
168
+ "irregularity_type": "Bid rigging and collusion",
169
+ "finding":
170
+ "Single-bid contracts awarded without competition. "
171
+ "Repeated winners across multiple tenders.",
172
+ "source": "CAG India (sample)",
173
+ "scraped_at": datetime.now().isoformat(),
174
+ "entity_type": "audit_report",
175
+ "alert_keywords": ["collusion", "irregularities"],
176
+ },
177
+ ]
178
+
179
+ def fetch_and_save(self, save: bool = True) -> list:
180
+ """Fetch all reports and save to disk."""
181
+ reports = self.fetch_report_list(limit=30)
182
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
183
+
184
+ if save and reports:
185
+ filepath = f"data/samples/cag_reports_{timestamp}.json"
186
+ self.save_json(reports, filepath)
187
+ logger.success(f"[CAG] Saved {len(reports)} reports to {filepath}")
188
+
189
+ return reports
190
+
191
+ def get_high_value_irregularities(self, reports: list,
192
+ min_crore: float = 10.0) -> list:
193
+ """
194
+ Filter reports where amount involved >= min_crore.
195
+ Helps prioritize large-scale fraud for investigation.
196
+ """
197
+ flagged = [
198
+ r for r in reports
199
+ if float(r.get("amount_crore", 0) or 0) >= min_crore
200
+ ]
201
+ logger.info(f"[CAG] Reports >= β‚Ή{min_crore}Cr: {len(flagged)}")
202
+ return flagged
203
+
204
+
205
+ # ── Run directly to test ──────────────────────────────────────────────────────
206
+ if __name__ == "__main__":
207
+ print("=" * 60)
208
+ print("BharatGraph - CAG Audit Report Scraper Test")
209
+ print("=" * 60)
210
+
211
+ scraper = CAGScraper()
212
+
213
+ print("\n[1] Fetching CAG audit report list...")
214
+ reports = scraper.fetch_report_list(limit=20)
215
+ print(f" Total reports found: {len(reports)}")
216
+
217
+ if reports:
218
+ r = reports[0]
219
+ print(f"\n Example report:")
220
+ print(f" Title: {r.get('title', 'N/A')[:60]}")
221
+ print(f" URL: {r.get('url', 'N/A')[:60]}")
222
+ print(f" Alert kws: {r.get('alert_keywords', [])}")
223
+ if r.get("amount_crore"):
224
+ print(f" Amount: β‚Ή{r.get('amount_crore')} Crore")
225
+
226
+ print("\n[2] Filtering high-value irregularities (>=10 Cr)...")
227
+ big = scraper.get_high_value_irregularities(reports, min_crore=10.0)
228
+ print(f" High-value reports: {len(big)}")
229
+
230
+ print("\n[3] Saving all reports...")
231
+ all_reports = scraper.fetch_and_save(save=True)
232
+ print(f" Saved {len(all_reports)} reports to data/samples/")
233
+ print("\nDone!")