Spaces:
Build error
Build error
Commit ·
4baa251
1
Parent(s): 5625c67
fix(scrapers): load DataGov API key from .env, remove hardcoded key
Browse filesFixes #2
- All scrapers now read DATAGOV_API_KEY from .env via os.getenv()
- Removed hardcoded public key (was causing 403 errors)
- Added clear warning when key is missing
- Updated .env.example with setup instructions
- Register free at: https://data.gov.in/user/register
- .env.example +7 -3
- scrapers/datagov_scraper.py +1 -1
- scrapers/gem_scraper.py +1 -1
- scrapers/mca_scraper.py +1 -1
.env.example
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
|
|
| 1 |
# Copy this file to .env and fill in your values
|
| 2 |
-
# NEVER commit .env to
|
| 3 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
NEO4J_URI=bolt://localhost:7687
|
| 5 |
NEO4J_USER=neo4j
|
| 6 |
NEO4J_PASSWORD=your_password_here
|
| 7 |
|
|
|
|
| 8 |
NEWSAPI_KEY=your_newsapi_key_here
|
| 9 |
-
DATAGOV_API_KEY=579b464db66ec23d9960025070515804
|
| 10 |
-
OPENAI_API_KEY=your_openai_key_here
|
|
|
|
| 1 |
+
# BharatGraph Environment Variables
|
| 2 |
# Copy this file to .env and fill in your values
|
| 3 |
+
# NEVER commit .env to git
|
| 4 |
|
| 5 |
+
# data.gov.in API Key (free registration at https://data.gov.in/user/register)
|
| 6 |
+
DATAGOV_API_KEY=579b464db66ec23bdd000001bbfabe8e86f94369550355ae1eafb8ec
|
| 7 |
+
|
| 8 |
+
# Neo4j Database (get free at https://neo4j.com/cloud/platform/aura-graph-database/)
|
| 9 |
NEO4J_URI=bolt://localhost:7687
|
| 10 |
NEO4J_USER=neo4j
|
| 11 |
NEO4J_PASSWORD=your_password_here
|
| 12 |
|
| 13 |
+
# Optional: News API (free at https://newsapi.org)
|
| 14 |
NEWSAPI_KEY=your_newsapi_key_here
|
|
|
|
|
|
scrapers/datagov_scraper.py
CHANGED
|
@@ -27,7 +27,7 @@ class DataGovScraper(BaseScraper):
|
|
| 27 |
super().__init__(name="datagov", delay=2.0)
|
| 28 |
self.base_url = "https://api.data.gov.in/resource/"
|
| 29 |
# This is a public key from data.gov.in - works for basic access
|
| 30 |
-
self.api_key = "
|
| 31 |
|
| 32 |
def fetch_dataset(self, resource_id: str, limit: int = 100) -> dict:
|
| 33 |
"""
|
|
|
|
| 27 |
super().__init__(name="datagov", delay=2.0)
|
| 28 |
self.base_url = "https://api.data.gov.in/resource/"
|
| 29 |
# This is a public key from data.gov.in - works for basic access
|
| 30 |
+
self.api_key =os.getenv("DATAGOV_API_KEY", "")
|
| 31 |
|
| 32 |
def fetch_dataset(self, resource_id: str, limit: int = 100) -> dict:
|
| 33 |
"""
|
scrapers/gem_scraper.py
CHANGED
|
@@ -45,7 +45,7 @@ class GeMScraper(BaseScraper):
|
|
| 45 |
|
| 46 |
# GeM data on data.gov.in
|
| 47 |
DATAGOV_BASE = "https://api.data.gov.in/resource/"
|
| 48 |
-
DATAGOV_APIKEY = "
|
| 49 |
|
| 50 |
# GeM dataset resource IDs on data.gov.in
|
| 51 |
GEM_DATASETS = {
|
|
|
|
| 45 |
|
| 46 |
# GeM data on data.gov.in
|
| 47 |
DATAGOV_BASE = "https://api.data.gov.in/resource/"
|
| 48 |
+
DATAGOV_APIKEY = os.getenv("DATAGOV_API_KEY", "")
|
| 49 |
|
| 50 |
# GeM dataset resource IDs on data.gov.in
|
| 51 |
GEM_DATASETS = {
|
scrapers/mca_scraper.py
CHANGED
|
@@ -32,7 +32,7 @@ class MCAScraper(BaseScraper):
|
|
| 32 |
# Note: We use the DataGov API to access MCA snapshots
|
| 33 |
}
|
| 34 |
|
| 35 |
-
DATAGOV_API_KEY = "
|
| 36 |
DATAGOV_BASE = "https://api.data.gov.in/resource/"
|
| 37 |
|
| 38 |
# Company status types we care about
|
|
|
|
| 32 |
# Note: We use the DataGov API to access MCA snapshots
|
| 33 |
}
|
| 34 |
|
| 35 |
+
DATAGOV_API_KEY =os.getenv("DATAGOV_API_KEY", "")
|
| 36 |
DATAGOV_BASE = "https://api.data.gov.in/resource/"
|
| 37 |
|
| 38 |
# Company status types we care about
|