going back to start
Browse files- vianu/spock/src/scraping.py +21 -20
vianu/spock/src/scraping.py
CHANGED
|
@@ -16,7 +16,6 @@ from io import BytesIO
|
|
| 16 |
import logging
|
| 17 |
import re
|
| 18 |
from typing import List
|
| 19 |
-
import urllib
|
| 20 |
import xml.etree.ElementTree as ET # nosec
|
| 21 |
|
| 22 |
import aiohttp
|
|
@@ -69,15 +68,14 @@ class Scraper(ABC):
|
|
| 69 |
return chunks
|
| 70 |
|
| 71 |
@staticmethod
|
| 72 |
-
async def _aiohttp_get_html(
|
|
|
|
|
|
|
| 73 |
"""Get the content of a given URL by an aiohttp GET request."""
|
| 74 |
async with aiohttp.ClientSession(headers=headers) as session:
|
| 75 |
async with session.get(url=url, params=params) as response:
|
| 76 |
-
print('------------------------------')
|
| 77 |
-
print('Status:', response.status)
|
| 78 |
response.raise_for_status()
|
| 79 |
text = await response.text()
|
| 80 |
-
print('HTML:', text[:1000])
|
| 81 |
return text
|
| 82 |
|
| 83 |
|
|
@@ -339,16 +337,10 @@ class EMAScraper(Scraper):
|
|
| 339 |
_source_favicon_url = (
|
| 340 |
"https://www.ema.europa.eu/themes/custom/ema_theme/favicon.ico"
|
| 341 |
)
|
| 342 |
-
|
| 343 |
-
# _pdf_search_template = (
|
| 344 |
-
# "https://www.ema.europa.eu/en/search?search_api_fulltext={term}"
|
| 345 |
-
# "&f%5B0%5D=ema_search_custom_entity_bundle%3Adocument" # This part is added to only retrieve PDF documents
|
| 346 |
-
# "&f%5B1%5D=ema_search_entity_is_document%3ADocument"
|
| 347 |
-
# )
|
| 348 |
_search_url = "https://www.ema.europa.eu/en/search"
|
| 349 |
_search_params = {
|
| 350 |
"search_api_fulltext": None,
|
| 351 |
-
"f[0]": "ema_search_custom_entity_bundle:document",
|
| 352 |
"f[1]": "ema_search_entity_is_document:Document",
|
| 353 |
}
|
| 354 |
_headers = {
|
|
@@ -389,14 +381,15 @@ class EMAScraper(Scraper):
|
|
| 389 |
"""Search the EMA database for PDF documents with a given term."""
|
| 390 |
|
| 391 |
# Get initial search results
|
| 392 |
-
# url = self._pdf_search_template.format(term=term)
|
| 393 |
-
# self.logger.debug(f"search ema database with url={url}")
|
| 394 |
params = deepcopy(self._search_params)
|
| 395 |
params["search_api_fulltext"] = term
|
| 396 |
-
log_url = f
|
|
|
|
|
|
|
| 397 |
self.logger.debug(f"search ema database with url={log_url}")
|
| 398 |
-
|
| 399 |
-
|
|
|
|
| 400 |
soup = BeautifulSoup(content, "html.parser")
|
| 401 |
|
| 402 |
# Get the number of search results and number of pages
|
|
@@ -416,7 +409,9 @@ class EMAScraper(Scraper):
|
|
| 416 |
# url = f"{url}&page={i}"
|
| 417 |
params["page"] = i
|
| 418 |
content = await self._aiohttp_get_html(
|
| 419 |
-
url=self._search_url,
|
|
|
|
|
|
|
| 420 |
)
|
| 421 |
soup = BeautifulSoup(content, "html.parser")
|
| 422 |
|
|
@@ -590,6 +585,8 @@ class MHRAScraper(Scraper):
|
|
| 590 |
_source_base_url = "https://www.gov.uk"
|
| 591 |
_language = "en"
|
| 592 |
|
|
|
|
|
|
|
| 593 |
def _extract_search_results_count(self, parent: Tag) -> int | None:
|
| 594 |
"""Extract the number of search results."""
|
| 595 |
div = parent.find("div", class_="result-info__header")
|
|
@@ -615,9 +612,13 @@ class MHRAScraper(Scraper):
|
|
| 615 |
self.logger.debug(f"search mhra's drug safety update database with url={url}")
|
| 616 |
content = await self._aiohttp_get_html(url=url)
|
| 617 |
soup = BeautifulSoup(content, "html.parser")
|
| 618 |
-
search_results = soup.select(
|
|
|
|
|
|
|
| 619 |
if len(search_results) != 1:
|
| 620 |
-
raise ValueError(
|
|
|
|
|
|
|
| 621 |
parent = search_results[0]
|
| 622 |
|
| 623 |
# Extract the number of search results
|
|
|
|
| 16 |
import logging
|
| 17 |
import re
|
| 18 |
from typing import List
|
|
|
|
| 19 |
import xml.etree.ElementTree as ET # nosec
|
| 20 |
|
| 21 |
import aiohttp
|
|
|
|
| 68 |
return chunks
|
| 69 |
|
| 70 |
@staticmethod
|
| 71 |
+
async def _aiohttp_get_html(
|
| 72 |
+
url: str, headers: dict | None = None, params: dict | None = None
|
| 73 |
+
) -> str:
|
| 74 |
"""Get the content of a given URL by an aiohttp GET request."""
|
| 75 |
async with aiohttp.ClientSession(headers=headers) as session:
|
| 76 |
async with session.get(url=url, params=params) as response:
|
|
|
|
|
|
|
| 77 |
response.raise_for_status()
|
| 78 |
text = await response.text()
|
|
|
|
| 79 |
return text
|
| 80 |
|
| 81 |
|
|
|
|
| 337 |
_source_favicon_url = (
|
| 338 |
"https://www.ema.europa.eu/themes/custom/ema_theme/favicon.ico"
|
| 339 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
_search_url = "https://www.ema.europa.eu/en/search"
|
| 341 |
_search_params = {
|
| 342 |
"search_api_fulltext": None,
|
| 343 |
+
"f[0]": "ema_search_custom_entity_bundle:document", # This part is added to only retrieve PDF documents
|
| 344 |
"f[1]": "ema_search_entity_is_document:Document",
|
| 345 |
}
|
| 346 |
_headers = {
|
|
|
|
| 381 |
"""Search the EMA database for PDF documents with a given term."""
|
| 382 |
|
| 383 |
# Get initial search results
|
|
|
|
|
|
|
| 384 |
params = deepcopy(self._search_params)
|
| 385 |
params["search_api_fulltext"] = term
|
| 386 |
+
log_url = f"{self._search_url}?" + "&".join(
|
| 387 |
+
[f"{k}={v}" for k, v in params.items()]
|
| 388 |
+
)
|
| 389 |
self.logger.debug(f"search ema database with url={log_url}")
|
| 390 |
+
content = await self._aiohttp_get_html(
|
| 391 |
+
url=self._search_url, headers=self._headers, params=params
|
| 392 |
+
)
|
| 393 |
soup = BeautifulSoup(content, "html.parser")
|
| 394 |
|
| 395 |
# Get the number of search results and number of pages
|
|
|
|
| 409 |
# url = f"{url}&page={i}"
|
| 410 |
params["page"] = i
|
| 411 |
content = await self._aiohttp_get_html(
|
| 412 |
+
url=self._search_url,
|
| 413 |
+
headers=self._headers,
|
| 414 |
+
params=params,
|
| 415 |
)
|
| 416 |
soup = BeautifulSoup(content, "html.parser")
|
| 417 |
|
|
|
|
| 585 |
_source_base_url = "https://www.gov.uk"
|
| 586 |
_language = "en"
|
| 587 |
|
| 588 |
+
_robots_txt_url = "https://www.gov.uk/robots.txt"
|
| 589 |
+
|
| 590 |
def _extract_search_results_count(self, parent: Tag) -> int | None:
|
| 591 |
"""Extract the number of search results."""
|
| 592 |
div = parent.find("div", class_="result-info__header")
|
|
|
|
| 612 |
self.logger.debug(f"search mhra's drug safety update database with url={url}")
|
| 613 |
content = await self._aiohttp_get_html(url=url)
|
| 614 |
soup = BeautifulSoup(content, "html.parser")
|
| 615 |
+
search_results = soup.select(
|
| 616 |
+
"div.govuk-grid-column-two-thirds.js-live-search-results-block.filtered-results"
|
| 617 |
+
)
|
| 618 |
if len(search_results) != 1:
|
| 619 |
+
raise ValueError(
|
| 620 |
+
f"expected 1 search results block, but found {len(search_results)}"
|
| 621 |
+
)
|
| 622 |
parent = search_results[0]
|
| 623 |
|
| 624 |
# Extract the number of search results
|