abinazebinoy commited on
Commit
0e9b6d0
Β·
1 Parent(s): 563cd5e

feat(scrapers): add GeM government contracts scraper

Browse files

- Scrapes procurement data from Government e-Marketplace (gem.gov.in)
- Extracts: order ID, buyer org, seller name, GSTIN, amount, date
- find_repeated_winners() detects contract concentration (key risk flag)
- Sample data shows same company winning 2 contracts (pattern detection)
- Links to politician graph: seller company β†’ directors β†’ politicians
- Falls back to structured sample data when API unavailable

Files changed (1) hide show
  1. scrapers/gem_scraper.py +291 -0
scrapers/gem_scraper.py ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ BharatGraph - GeM (Government e-Marketplace) Scraper
3
+ Fetches government contract and procurement data.
4
+ Source: gem.gov.in + data.gov.in GeM datasets (public)
5
+
6
+ WHY THIS IS THE MOST IMPORTANT SCRAPER:
7
+ GeM is where the government buys everything.
8
+ If a politician's relative's company wins contracts here,
9
+ that's exactly the pattern BharatGraph needs to flag.
10
+
11
+ Pattern we're looking for:
12
+ Politician β†’ (relative/director) β†’ Company β†’ GeM Contract Won
13
+ """
14
+
15
+ import json
16
+ import re
17
+ from datetime import datetime
18
+ from scrapers.base_scraper import BaseScraper
19
+ from loguru import logger
20
+
21
+
22
+ class GeMScraper(BaseScraper):
23
+ """
24
+ Scrapes procurement contract data from Government e-Marketplace (GeM).
25
+
26
+ GeM is India's official online procurement portal.
27
+ All government purchases through GeM are public record.
28
+
29
+ Data includes:
30
+ - Contract/order details
31
+ - Buyer (government department)
32
+ - Seller (company that won)
33
+ - Contract value in INR
34
+ - Product/service category
35
+ - Order date
36
+
37
+ This is critical for finding:
38
+ - Repeated contract winners
39
+ - Companies linked to politicians winning contracts
40
+ - Unusual concentration of contracts
41
+ """
42
+
43
+ # GeM public stats API
44
+ GEM_STATS_URL = "https://mkp.gem.gov.in/api/v2/public/stats"
45
+
46
+ # GeM data on data.gov.in
47
+ DATAGOV_BASE = "https://api.data.gov.in/resource/"
48
+ DATAGOV_APIKEY = "579b464db66ec23d9960025070515804"
49
+
50
+ # GeM dataset resource IDs on data.gov.in
51
+ GEM_DATASETS = {
52
+ "gem_orders": "0a4b1e09-c3e0-4f0a-8f06-4f62a6d08e4e",
53
+ }
54
+
55
+ def __init__(self):
56
+ super().__init__(name="gem", delay=2.0)
57
+
58
+ def fetch_gem_stats(self) -> dict:
59
+ """
60
+ Fetch public GeM platform statistics.
61
+ Returns total orders, GMV, sellers, buyers count.
62
+ """
63
+ logger.info("[GeM] Fetching platform statistics...")
64
+ data = self.get_json(self.GEM_STATS_URL)
65
+
66
+ if data:
67
+ logger.success("[GeM] Stats fetched successfully")
68
+ return {
69
+ "source": "GeM Public Stats",
70
+ "scraped_at": datetime.now().isoformat(),
71
+ "stats": data,
72
+ }
73
+
74
+ logger.warning("[GeM] Could not fetch stats, using sample")
75
+ return self._get_sample_stats()
76
+
77
+ def fetch_contracts_by_ministry(self, ministry: str = "all",
78
+ limit: int = 50) -> list:
79
+ """
80
+ Fetch procurement contracts, optionally filtered by ministry.
81
+ Uses data.gov.in GeM dataset.
82
+ """
83
+ import os
84
+ api_key = os.getenv("DATAGOV_API_KEY", self.DATAGOV_APIKEY)
85
+ logger.info(f"[GeM] Fetching contracts for ministry: {ministry}")
86
+
87
+ url = f"{self.DATAGOV_BASE}{self.GEM_DATASETS['gem_orders']}"
88
+ params = {
89
+ "api-key": api_key,
90
+ "format": "json",
91
+ "limit": limit,
92
+ }
93
+ if ministry != "all":
94
+ params["filters[ministry_name]"] = ministry
95
+
96
+ data = self.get_json(url, params=params)
97
+
98
+ if data and "records" in data:
99
+ contracts = self._normalize_contracts(data["records"])
100
+ logger.success(f"[GeM] Got {len(contracts)} contracts")
101
+ return contracts
102
+
103
+ logger.warning("[GeM] API unavailable, using sample data")
104
+ return self._get_sample_contracts()
105
+
106
+ def _normalize_contracts(self, records: list) -> list:
107
+ """Normalize GeM contract records."""
108
+ contracts = []
109
+ for rec in records:
110
+ contract = {
111
+ "order_id": rec.get("order_id", rec.get("id", "")),
112
+ "buyer_org": rec.get("buyer_organisation",
113
+ rec.get("ministry", "")),
114
+ "seller_name": rec.get("seller_name",
115
+ rec.get("company_name", "")),
116
+ "seller_gstin": rec.get("gstin", ""),
117
+ "product": rec.get("product_name",
118
+ rec.get("category", "")),
119
+ "amount_inr": rec.get("order_value",
120
+ rec.get("amount", 0)),
121
+ "order_date": rec.get("order_date",
122
+ rec.get("date", "")),
123
+ "state": rec.get("buyer_state", ""),
124
+ "source": "GeM / data.gov.in",
125
+ "scraped_at": datetime.now().isoformat(),
126
+ "entity_type": "contract",
127
+ }
128
+ if contract["seller_name"] or contract["order_id"]:
129
+ contracts.append(contract)
130
+ return contracts
131
+
132
+ def _get_sample_stats(self) -> dict:
133
+ """Sample GeM statistics for testing."""
134
+ return {
135
+ "source": "GeM Sample Data",
136
+ "scraped_at": datetime.now().isoformat(),
137
+ "stats": {
138
+ "total_orders": 6200000,
139
+ "total_gmv_crore": 285000,
140
+ "total_sellers": 680000,
141
+ "total_buyers": 75000,
142
+ "note": "SAMPLE DATA - real stats at gem.gov.in/statistics",
143
+ },
144
+ }
145
+
146
+ def _get_sample_contracts(self) -> list:
147
+ """
148
+ Sample contract structures for graph testing.
149
+ Shows the type of data we'll eventually link to politicians.
150
+ """
151
+ return [
152
+ {
153
+ "order_id": "GEM/2024/B/12345",
154
+ "buyer_org": "Ministry of Rural Development",
155
+ "seller_name": "SAMPLE INFRASTRUCTURE PVT LTD",
156
+ "seller_gstin":"33AABCS1234A1Z5",
157
+ "product": "Construction Services",
158
+ "amount_inr": 15000000,
159
+ "amount_crore": 1.5,
160
+ "order_date": "2024-03-15",
161
+ "state": "Tamil Nadu",
162
+ "source": "GeM (sample)",
163
+ "scraped_at": datetime.now().isoformat(),
164
+ "entity_type": "contract",
165
+ "note": "SAMPLE - replace with real GeM fetch",
166
+ },
167
+ {
168
+ "order_id": "GEM/2024/B/67890",
169
+ "buyer_org": "Ministry of Health",
170
+ "seller_name": "SAMPLE MEDICAL SUPPLIES LTD",
171
+ "seller_gstin":"27AABCM5678B1Z3",
172
+ "product": "Medical Equipment",
173
+ "amount_inr": 50000000,
174
+ "amount_crore": 5.0,
175
+ "order_date": "2024-06-10",
176
+ "state": "Maharashtra",
177
+ "source": "GeM (sample)",
178
+ "scraped_at": datetime.now().isoformat(),
179
+ "entity_type": "contract",
180
+ "note": "SAMPLE - replace with real GeM fetch",
181
+ },
182
+ {
183
+ "order_id": "GEM/2024/B/11111",
184
+ "buyer_org": "Ministry of Rural Development",
185
+ "seller_name": "SAMPLE INFRASTRUCTURE PVT LTD",
186
+ "seller_gstin":"33AABCS1234A1Z5",
187
+ "product": "Road Construction",
188
+ "amount_inr": 80000000,
189
+ "amount_crore": 8.0,
190
+ "order_date": "2024-09-20",
191
+ "state": "Tamil Nadu",
192
+ "source": "GeM (sample)",
193
+ "scraped_at": datetime.now().isoformat(),
194
+ "entity_type": "contract",
195
+ "note": "SAMPLE - same company, 2nd contract (pattern!)",
196
+ },
197
+ ]
198
+
199
+ def find_repeated_winners(self, contracts: list) -> dict:
200
+ """
201
+ Find companies that won multiple contracts.
202
+ This is a KEY risk indicator - contract concentration.
203
+
204
+ Returns dict of {seller_name: [list of contracts]}
205
+ """
206
+ from collections import defaultdict
207
+ winners = defaultdict(list)
208
+
209
+ for c in contracts:
210
+ seller = c.get("seller_name", "").strip()
211
+ if seller:
212
+ winners[seller].append(c)
213
+
214
+ # Only return sellers with 2+ contracts
215
+ repeated = {k: v for k, v in winners.items() if len(v) >= 2}
216
+
217
+ logger.info(f"[GeM] Repeated contract winners: {len(repeated)}")
218
+ for seller, contracts_list in repeated.items():
219
+ total = sum(float(c.get("amount_crore", 0) or 0)
220
+ for c in contracts_list)
221
+ logger.info(
222
+ f" β†’ {seller}: {len(contracts_list)} contracts, "
223
+ f"β‚Ή{total:.2f} Cr total"
224
+ )
225
+ return repeated
226
+
227
+ def fetch_and_save(self, save: bool = True) -> dict:
228
+ """Fetch contracts + stats and save to disk."""
229
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
230
+
231
+ stats = self.fetch_gem_stats()
232
+ contracts = self.fetch_contracts_by_ministry(limit=30)
233
+ repeated = self.find_repeated_winners(contracts)
234
+
235
+ result = {
236
+ "stats": stats,
237
+ "contracts": contracts,
238
+ "repeated_winners": {
239
+ k: v for k, v in repeated.items()
240
+ },
241
+ "scraped_at": datetime.now().isoformat(),
242
+ }
243
+
244
+ if save:
245
+ filepath = f"data/samples/gem_contracts_{timestamp}.json"
246
+ self.save_json(result, filepath)
247
+ logger.success(f"[GeM] Saved to {filepath}")
248
+
249
+ return result
250
+
251
+
252
+ # ── Run directly to test ──────────────────────────────────────────────────────
253
+ if __name__ == "__main__":
254
+ print("=" * 60)
255
+ print("BharatGraph - GeM Contracts Scraper Test")
256
+ print("=" * 60)
257
+
258
+ scraper = GeMScraper()
259
+
260
+ print("\n[1] Fetching GeM platform stats...")
261
+ stats = scraper.fetch_gem_stats()
262
+ s = stats.get("stats", {})
263
+ print(f" Total orders: {s.get('total_orders', 'N/A'):,}")
264
+ print(f" Total GMV: β‚Ή{s.get('total_gmv_crore', 'N/A'):,} Cr")
265
+ print(f" Total sellers: {s.get('total_sellers', 'N/A'):,}")
266
+
267
+ print("\n[2] Fetching contracts...")
268
+ contracts = scraper.fetch_contracts_by_ministry(limit=20)
269
+ print(f" Total contracts: {len(contracts)}")
270
+
271
+ if contracts:
272
+ c = contracts[0]
273
+ print(f"\n Example contract:")
274
+ print(f" Order ID: {c.get('order_id')}")
275
+ print(f" Buyer: {c.get('buyer_org')}")
276
+ print(f" Seller: {c.get('seller_name')}")
277
+ print(f" Amount: β‚Ή{c.get('amount_crore')} Cr")
278
+ print(f" Date: {c.get('order_date')}")
279
+
280
+ print("\n[3] Finding repeated contract winners (risk indicator)...")
281
+ repeated = scraper.find_repeated_winners(contracts)
282
+ if repeated:
283
+ print(f" ⚠️ {len(repeated)} companies won multiple contracts!")
284
+ else:
285
+ print(" No repeated winners found in sample")
286
+
287
+ print("\n[4] Saving full data...")
288
+ result = scraper.fetch_and_save(save=True)
289
+ print(f" Contracts saved: {len(result['contracts'])}")
290
+ print(f" Repeated winners: {len(result['repeated_winners'])}")
291
+ print("\nDone! Check data/samples/ folder.")