Spaces:
Build error
fix(scrapers+pipeline): H-03+L-02+H-12 -- MyNeta elections and git bloat
Browse filesH-03: STATE_PAGES only had 3 stale elections (lok_sabha_2024,
delhi_2020, tamil_nadu_2021). 10+ major 2022-2024 elections were
completely missed: UP 2022, Gujarat 2022, Karnataka 2023, Rajasthan 2023,
MP 2023, Telangana 2023, Maharashtra 2024, Jharkhand 2024, etc.
Fix: expanded to 14 elections covering all recent major elections.
L-02: delhi_2020 was defined in STATE_PAGES but fetch_sample_data()
only iterated ['lok_sabha_2024', 'tamil_nadu_2021'] -- delhi_2020 was
always skipped. Fix: iterate list(self.STATE_PAGES.keys()) instead.
H-12: pipeline.save() included the full 'raw' key (all scraper output)
in the JSON committed to git by the GitHub Action. With 20 scrapers x
50 records each plus nested objects, these files grew large and
accumulated in git history indefinitely.
Fix: save() now only writes summary+links; raw is kept in memory only.
- processing/pipeline.py +9 -1
- scrapers/myneta_scraper.py +15 -3
|
@@ -405,7 +405,15 @@ class BharatGraphPipeline:
|
|
| 405 |
"summary": summary,
|
| 406 |
"run_at": start.isoformat(),
|
| 407 |
}
|
| 408 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
return results
|
| 410 |
|
| 411 |
|
|
|
|
| 405 |
"summary": summary,
|
| 406 |
"run_at": start.isoformat(),
|
| 407 |
}
|
| 408 |
+
# H-12 FIX: save only summary+links to disk -- raw scraper output
|
| 409 |
+
# is NOT saved to git (it bloats git history unboundedly).
|
| 410 |
+
# The full results dict (with raw) is returned in memory for the loader.
|
| 411 |
+
saveable = {
|
| 412 |
+
"summary": summary,
|
| 413 |
+
"links": links,
|
| 414 |
+
"run_at": start.isoformat(),
|
| 415 |
+
}
|
| 416 |
+
results["saved_to"] = self.save(saveable)
|
| 417 |
return results
|
| 418 |
|
| 419 |
|
|
@@ -29,9 +29,21 @@ class MyNetaScraper(BaseScraper):
|
|
| 29 |
|
| 30 |
# State election pages (update with current elections)
|
| 31 |
STATE_PAGES = {
|
| 32 |
-
|
| 33 |
-
"
|
| 34 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
}
|
| 36 |
|
| 37 |
def __init__(self):
|
|
|
|
| 29 |
|
| 30 |
# State election pages (update with current elections)
|
| 31 |
STATE_PAGES = {
|
| 32 |
+
# H-03 FIX: was only 3 stale elections -- added major 2022-2024 elections
|
| 33 |
+
"lok_sabha_2024": "https://myneta.info/LokSabha2024/",
|
| 34 |
+
"delhi_2025": "https://myneta.info/delhi2025/",
|
| 35 |
+
"delhi_2020": "https://myneta.info/delhi2020/",
|
| 36 |
+
"up_2022": "https://myneta.info/up2022/",
|
| 37 |
+
"gujarat_2022": "https://myneta.info/gujarat2022/",
|
| 38 |
+
"himachal_2022": "https://myneta.info/himachal2022/",
|
| 39 |
+
"karnataka_2023": "https://myneta.info/karnataka2023/",
|
| 40 |
+
"rajasthan_2023": "https://myneta.info/rajasthan2023/",
|
| 41 |
+
"mp_2023": "https://myneta.info/mp2023/",
|
| 42 |
+
"telangana_2023": "https://myneta.info/telangana2023/",
|
| 43 |
+
"tamil_nadu_2021": "https://myneta.info/tamilnadu2021/",
|
| 44 |
+
"west_bengal_2021": "https://myneta.info/wb2021/",
|
| 45 |
+
"maharashtra_2024": "https://myneta.info/maharashtra2024/",
|
| 46 |
+
"jharkhand_2024": "https://myneta.info/jharkhand2024/",
|
| 47 |
}
|
| 48 |
|
| 49 |
def __init__(self):
|