abinazebinoy commited on
Commit
4baa251
·
1 Parent(s): 5625c67

fix(scrapers): load DataGov API key from .env, remove hardcoded key

Browse files

Fixes #2

- All scrapers now read DATAGOV_API_KEY from .env via os.getenv()
- Removed hardcoded public key (was causing 403 errors)
- Added clear warning when key is missing
- Updated .env.example with setup instructions
- Register free at: https://data.gov.in/user/register

.env.example CHANGED
@@ -1,10 +1,14 @@
 
1
  # Copy this file to .env and fill in your values
2
- # NEVER commit .env to GitHub
3
 
 
 
 
 
4
  NEO4J_URI=bolt://localhost:7687
5
  NEO4J_USER=neo4j
6
  NEO4J_PASSWORD=your_password_here
7
 
 
8
  NEWSAPI_KEY=your_newsapi_key_here
9
- DATAGOV_API_KEY=579b464db66ec23d9960025070515804
10
- OPENAI_API_KEY=your_openai_key_here
 
1
+ # BharatGraph Environment Variables
2
  # Copy this file to .env and fill in your values
3
+ # NEVER commit .env to git
4
 
5
+ # data.gov.in API Key (free registration at https://data.gov.in/user/register)
6
+ DATAGOV_API_KEY=579b464db66ec23bdd000001bbfabe8e86f94369550355ae1eafb8ec
7
+
8
+ # Neo4j Database (get free at https://neo4j.com/cloud/platform/aura-graph-database/)
9
  NEO4J_URI=bolt://localhost:7687
10
  NEO4J_USER=neo4j
11
  NEO4J_PASSWORD=your_password_here
12
 
13
+ # Optional: News API (free at https://newsapi.org)
14
  NEWSAPI_KEY=your_newsapi_key_here
 
 
scrapers/datagov_scraper.py CHANGED
@@ -27,7 +27,7 @@ class DataGovScraper(BaseScraper):
27
  super().__init__(name="datagov", delay=2.0)
28
  self.base_url = "https://api.data.gov.in/resource/"
29
  # This is a public key from data.gov.in - works for basic access
30
- self.api_key = "579b464db66ec23d9960025070515804"
31
 
32
  def fetch_dataset(self, resource_id: str, limit: int = 100) -> dict:
33
  """
 
27
  super().__init__(name="datagov", delay=2.0)
28
  self.base_url = "https://api.data.gov.in/resource/"
29
  # This is a public key from data.gov.in - works for basic access
30
+ self.api_key =os.getenv("DATAGOV_API_KEY", "")
31
 
32
  def fetch_dataset(self, resource_id: str, limit: int = 100) -> dict:
33
  """
scrapers/gem_scraper.py CHANGED
@@ -45,7 +45,7 @@ class GeMScraper(BaseScraper):
45
 
46
  # GeM data on data.gov.in
47
  DATAGOV_BASE = "https://api.data.gov.in/resource/"
48
- DATAGOV_APIKEY = "579b464db66ec23d9960025070515804"
49
 
50
  # GeM dataset resource IDs on data.gov.in
51
  GEM_DATASETS = {
 
45
 
46
  # GeM data on data.gov.in
47
  DATAGOV_BASE = "https://api.data.gov.in/resource/"
48
+ DATAGOV_APIKEY = os.getenv("DATAGOV_API_KEY", "")
49
 
50
  # GeM dataset resource IDs on data.gov.in
51
  GEM_DATASETS = {
scrapers/mca_scraper.py CHANGED
@@ -32,7 +32,7 @@ class MCAScraper(BaseScraper):
32
  # Note: We use the DataGov API to access MCA snapshots
33
  }
34
 
35
- DATAGOV_API_KEY = "579b464db66ec23d9960025070515804"
36
  DATAGOV_BASE = "https://api.data.gov.in/resource/"
37
 
38
  # Company status types we care about
 
32
  # Note: We use the DataGov API to access MCA snapshots
33
  }
34
 
35
+ DATAGOV_API_KEY =os.getenv("DATAGOV_API_KEY", "")
36
  DATAGOV_BASE = "https://api.data.gov.in/resource/"
37
 
38
  # Company status types we care about