jaegglic commited on
Commit
9d3be6a
·
1 Parent(s): c19c6ec

going back to start

Browse files
Files changed (1) hide show
  1. vianu/spock/src/scraping.py +21 -20
vianu/spock/src/scraping.py CHANGED
@@ -16,7 +16,6 @@ from io import BytesIO
16
  import logging
17
  import re
18
  from typing import List
19
- import urllib
20
  import xml.etree.ElementTree as ET # nosec
21
 
22
  import aiohttp
@@ -69,15 +68,14 @@ class Scraper(ABC):
69
  return chunks
70
 
71
  @staticmethod
72
- async def _aiohttp_get_html(url: str, headers: dict | None = None, params: dict | None = None) -> str:
 
 
73
  """Get the content of a given URL by an aiohttp GET request."""
74
  async with aiohttp.ClientSession(headers=headers) as session:
75
  async with session.get(url=url, params=params) as response:
76
- print('------------------------------')
77
- print('Status:', response.status)
78
  response.raise_for_status()
79
  text = await response.text()
80
- print('HTML:', text[:1000])
81
  return text
82
 
83
 
@@ -339,16 +337,10 @@ class EMAScraper(Scraper):
339
  _source_favicon_url = (
340
  "https://www.ema.europa.eu/themes/custom/ema_theme/favicon.ico"
341
  )
342
-
343
- # _pdf_search_template = (
344
- # "https://www.ema.europa.eu/en/search?search_api_fulltext={term}"
345
- # "&f%5B0%5D=ema_search_custom_entity_bundle%3Adocument" # This part is added to only retrieve PDF documents
346
- # "&f%5B1%5D=ema_search_entity_is_document%3ADocument"
347
- # )
348
  _search_url = "https://www.ema.europa.eu/en/search"
349
  _search_params = {
350
  "search_api_fulltext": None,
351
- "f[0]": "ema_search_custom_entity_bundle:document", # This part is added to only retrieve PDF documents
352
  "f[1]": "ema_search_entity_is_document:Document",
353
  }
354
  _headers = {
@@ -389,14 +381,15 @@ class EMAScraper(Scraper):
389
  """Search the EMA database for PDF documents with a given term."""
390
 
391
  # Get initial search results
392
- # url = self._pdf_search_template.format(term=term)
393
- # self.logger.debug(f"search ema database with url={url}")
394
  params = deepcopy(self._search_params)
395
  params["search_api_fulltext"] = term
396
- log_url = f'{self._search_url}?' + "&".join([f"{k}={v}" for k, v in params.items()])
 
 
397
  self.logger.debug(f"search ema database with url={log_url}")
398
- # self.logger.debug(f"search ema database with url={self._search_url} and params={params}")
399
- content = await self._aiohttp_get_html(url=self._search_url, headers=self._headers, params=params)
 
400
  soup = BeautifulSoup(content, "html.parser")
401
 
402
  # Get the number of search results and number of pages
@@ -416,7 +409,9 @@ class EMAScraper(Scraper):
416
  # url = f"{url}&page={i}"
417
  params["page"] = i
418
  content = await self._aiohttp_get_html(
419
- url=self._search_url, headers=self._headers, params=params,
 
 
420
  )
421
  soup = BeautifulSoup(content, "html.parser")
422
 
@@ -590,6 +585,8 @@ class MHRAScraper(Scraper):
590
  _source_base_url = "https://www.gov.uk"
591
  _language = "en"
592
 
 
 
593
  def _extract_search_results_count(self, parent: Tag) -> int | None:
594
  """Extract the number of search results."""
595
  div = parent.find("div", class_="result-info__header")
@@ -615,9 +612,13 @@ class MHRAScraper(Scraper):
615
  self.logger.debug(f"search mhra's drug safety update database with url={url}")
616
  content = await self._aiohttp_get_html(url=url)
617
  soup = BeautifulSoup(content, "html.parser")
618
- search_results = soup.select('div.govuk-grid-column-two-thirds.js-live-search-results-block.filtered-results')
 
 
619
  if len(search_results) != 1:
620
- raise ValueError(f"expected 1 search results block, but found {len(search_results)}")
 
 
621
  parent = search_results[0]
622
 
623
  # Extract the number of search results
 
16
  import logging
17
  import re
18
  from typing import List
 
19
  import xml.etree.ElementTree as ET # nosec
20
 
21
  import aiohttp
 
68
  return chunks
69
 
70
  @staticmethod
71
+ async def _aiohttp_get_html(
72
+ url: str, headers: dict | None = None, params: dict | None = None
73
+ ) -> str:
74
  """Get the content of a given URL by an aiohttp GET request."""
75
  async with aiohttp.ClientSession(headers=headers) as session:
76
  async with session.get(url=url, params=params) as response:
 
 
77
  response.raise_for_status()
78
  text = await response.text()
 
79
  return text
80
 
81
 
 
337
  _source_favicon_url = (
338
  "https://www.ema.europa.eu/themes/custom/ema_theme/favicon.ico"
339
  )
 
 
 
 
 
 
340
  _search_url = "https://www.ema.europa.eu/en/search"
341
  _search_params = {
342
  "search_api_fulltext": None,
343
+ "f[0]": "ema_search_custom_entity_bundle:document", # This part is added to only retrieve PDF documents
344
  "f[1]": "ema_search_entity_is_document:Document",
345
  }
346
  _headers = {
 
381
  """Search the EMA database for PDF documents with a given term."""
382
 
383
  # Get initial search results
 
 
384
  params = deepcopy(self._search_params)
385
  params["search_api_fulltext"] = term
386
+ log_url = f"{self._search_url}?" + "&".join(
387
+ [f"{k}={v}" for k, v in params.items()]
388
+ )
389
  self.logger.debug(f"search ema database with url={log_url}")
390
+ content = await self._aiohttp_get_html(
391
+ url=self._search_url, headers=self._headers, params=params
392
+ )
393
  soup = BeautifulSoup(content, "html.parser")
394
 
395
  # Get the number of search results and number of pages
 
409
  # url = f"{url}&page={i}"
410
  params["page"] = i
411
  content = await self._aiohttp_get_html(
412
+ url=self._search_url,
413
+ headers=self._headers,
414
+ params=params,
415
  )
416
  soup = BeautifulSoup(content, "html.parser")
417
 
 
585
  _source_base_url = "https://www.gov.uk"
586
  _language = "en"
587
 
588
+ _robots_txt_url = "https://www.gov.uk/robots.txt"
589
+
590
  def _extract_search_results_count(self, parent: Tag) -> int | None:
591
  """Extract the number of search results."""
592
  div = parent.find("div", class_="result-info__header")
 
612
  self.logger.debug(f"search mhra's drug safety update database with url={url}")
613
  content = await self._aiohttp_get_html(url=url)
614
  soup = BeautifulSoup(content, "html.parser")
615
+ search_results = soup.select(
616
+ "div.govuk-grid-column-two-thirds.js-live-search-results-block.filtered-results"
617
+ )
618
  if len(search_results) != 1:
619
+ raise ValueError(
620
+ f"expected 1 search results block, but found {len(search_results)}"
621
+ )
622
  parent = search_results[0]
623
 
624
  # Extract the number of search results