Buckets:
| """Functionality for working with InvenioRDM instances like Zenodo.""" | |
| import os | |
| from functools import partial | |
| from typing import Literal | |
| import dotenv | |
| import requests | |
| from requests.exceptions import HTTPError | |
| import calkit | |
| ServiceName = Literal["zenodo", "caltechdata"] | |
| DEFAULT_SERVICE = "zenodo" | |
| def get_token(service: ServiceName = DEFAULT_SERVICE) -> str: | |
| dotenv.load_dotenv() | |
| config = calkit.config.read() | |
| if service == "zenodo": | |
| token = config.zenodo_token | |
| if token is None: | |
| token = os.getenv("ZENODO_TOKEN") | |
| if token is None: | |
| token = calkit.hub.get("/user/zenodo-token")["access_token"] | |
| return token | |
| elif service == "caltechdata": | |
| token = config.caltechdata_token | |
| if token is None: | |
| raise ValueError(f"No token for {service} found") | |
| return token | |
| else: | |
| raise ValueError(f"Unknown archival service '{service}'") | |
| def get_base_url(service: ServiceName = DEFAULT_SERVICE) -> str: | |
| override = os.getenv(f"CALKIT_INVENIO_BASE_URL_{service.upper()}") | |
| if override: | |
| return override | |
| current_env = calkit.config.get_env() | |
| if service == "zenodo": | |
| if ( | |
| current_env in ["local", "test"] | |
| and not os.getenv("CALKIT_USE_PROD_FOR_TESTS") == "1" | |
| ): | |
| return "https://sandbox.zenodo.org/api" | |
| return "https://zenodo.org/api" | |
| elif service == "caltechdata": | |
| if ( | |
| current_env in ["local", "test"] | |
| and not os.getenv("CALKIT_USE_PROD_FOR_TESTS") == "1" | |
| ): | |
| return "https://data.caltechlibrary.dev/api" | |
| else: | |
| return "https://data.caltech.edu/api" | |
| else: | |
| raise ValueError(f"Unknown archival service '{service}'") | |
| def _request( | |
| kind: Literal["get", "post", "put", "patch", "delete"], | |
| path: str, | |
| params: dict | None = None, | |
| json: dict | list | None = None, | |
| data: dict | bytes | None = None, | |
| headers: dict | None = None, | |
| auth: bool = True, | |
| as_json: bool = True, | |
| service: ServiceName = DEFAULT_SERVICE, | |
| **kwargs, | |
| ): | |
| if params is None: | |
| params = {} | |
| if auth and "access_token" not in params: | |
| params = params | {"access_token": get_token(service=service)} | |
| func = getattr(requests, kind) | |
| resp = func( | |
| get_base_url(service=service) + path, | |
| params=params, | |
| json=json, | |
| data=data, | |
| headers=headers, | |
| **kwargs, | |
| ) | |
| if resp.status_code >= 400: | |
| msg = f"{resp.status_code}: " | |
| try: | |
| resp_json = resp.json() | |
| if "message" in resp_json: | |
| msg += resp_json["message"] | |
| if "errors" in resp_json: | |
| msg += f"\nErrors:\n{resp_json['errors']}" | |
| except ValueError: | |
| msg += resp.text | |
| raise HTTPError(msg) | |
| resp.raise_for_status() | |
| if as_json: | |
| return resp.json() | |
| else: | |
| return resp | |
| get = partial(_request, "get") | |
| post = partial(_request, "post") | |
| patch = partial(_request, "patch") | |
| put = partial(_request, "put") | |
| delete = partial(_request, "delete") | |
| def extract_doi(record: dict) -> str | None: | |
| """Extract the DOI identifier from an InvenioRDM record or response. | |
| Depending on the endpoint and whether the DOI has been minted yet, the | |
| identifier can live in different places, so check all known locations and | |
| return ``None`` if it isn't present yet. | |
| """ | |
| pids = record.get("pids") | |
| if isinstance(pids, dict): | |
| doi = pids.get("doi") | |
| if isinstance(doi, dict) and doi.get("identifier"): | |
| return doi["identifier"] | |
| if record.get("doi"): | |
| return record["doi"] | |
| metadata = record.get("metadata") | |
| if isinstance(metadata, dict) and metadata.get("doi"): | |
| return metadata["doi"] | |
| return None | |
| def get_download_urls( | |
| record_id: int | str, | |
| service: ServiceName = DEFAULT_SERVICE, | |
| auth: bool = True, | |
| ) -> dict[str, str]: | |
| resp = get(f"/records/{record_id}", service=service, auth=auth) | |
| download_urls = [f["links"]["self"] for f in resp["files"]] | |
| filenames = [f["key"] for f in resp["files"]] | |
| urls = {} | |
| for fname, url in zip(filenames, download_urls): | |
| urls[fname] = url | |
| return urls | |
Xet Storage Details
- Size:
- 4.29 kB
- Xet hash:
- 1cf356ceaa1e0bc892b4d2ef933391c55c6911d0b859dacf2442694f08d633d4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.