Buckets:

download
raw
4.29 kB
"""Functionality for working with InvenioRDM instances like Zenodo."""
import os
from functools import partial
from typing import Literal
import dotenv
import requests
from requests.exceptions import HTTPError
import calkit
ServiceName = Literal["zenodo", "caltechdata"]
DEFAULT_SERVICE = "zenodo"
def get_token(service: ServiceName = DEFAULT_SERVICE) -> str:
dotenv.load_dotenv()
config = calkit.config.read()
if service == "zenodo":
token = config.zenodo_token
if token is None:
token = os.getenv("ZENODO_TOKEN")
if token is None:
token = calkit.hub.get("/user/zenodo-token")["access_token"]
return token
elif service == "caltechdata":
token = config.caltechdata_token
if token is None:
raise ValueError(f"No token for {service} found")
return token
else:
raise ValueError(f"Unknown archival service '{service}'")
def get_base_url(service: ServiceName = DEFAULT_SERVICE) -> str:
override = os.getenv(f"CALKIT_INVENIO_BASE_URL_{service.upper()}")
if override:
return override
current_env = calkit.config.get_env()
if service == "zenodo":
if (
current_env in ["local", "test"]
and not os.getenv("CALKIT_USE_PROD_FOR_TESTS") == "1"
):
return "https://sandbox.zenodo.org/api"
return "https://zenodo.org/api"
elif service == "caltechdata":
if (
current_env in ["local", "test"]
and not os.getenv("CALKIT_USE_PROD_FOR_TESTS") == "1"
):
return "https://data.caltechlibrary.dev/api"
else:
return "https://data.caltech.edu/api"
else:
raise ValueError(f"Unknown archival service '{service}'")
def _request(
kind: Literal["get", "post", "put", "patch", "delete"],
path: str,
params: dict | None = None,
json: dict | list | None = None,
data: dict | bytes | None = None,
headers: dict | None = None,
auth: bool = True,
as_json: bool = True,
service: ServiceName = DEFAULT_SERVICE,
**kwargs,
):
if params is None:
params = {}
if auth and "access_token" not in params:
params = params | {"access_token": get_token(service=service)}
func = getattr(requests, kind)
resp = func(
get_base_url(service=service) + path,
params=params,
json=json,
data=data,
headers=headers,
**kwargs,
)
if resp.status_code >= 400:
msg = f"{resp.status_code}: "
try:
resp_json = resp.json()
if "message" in resp_json:
msg += resp_json["message"]
if "errors" in resp_json:
msg += f"\nErrors:\n{resp_json['errors']}"
except ValueError:
msg += resp.text
raise HTTPError(msg)
resp.raise_for_status()
if as_json:
return resp.json()
else:
return resp
get = partial(_request, "get")
post = partial(_request, "post")
patch = partial(_request, "patch")
put = partial(_request, "put")
delete = partial(_request, "delete")
def extract_doi(record: dict) -> str | None:
"""Extract the DOI identifier from an InvenioRDM record or response.
Depending on the endpoint and whether the DOI has been minted yet, the
identifier can live in different places, so check all known locations and
return ``None`` if it isn't present yet.
"""
pids = record.get("pids")
if isinstance(pids, dict):
doi = pids.get("doi")
if isinstance(doi, dict) and doi.get("identifier"):
return doi["identifier"]
if record.get("doi"):
return record["doi"]
metadata = record.get("metadata")
if isinstance(metadata, dict) and metadata.get("doi"):
return metadata["doi"]
return None
def get_download_urls(
record_id: int | str,
service: ServiceName = DEFAULT_SERVICE,
auth: bool = True,
) -> dict[str, str]:
resp = get(f"/records/{record_id}", service=service, auth=auth)
download_urls = [f["links"]["self"] for f in resp["files"]]
filenames = [f["key"] for f in resp["files"]]
urls = {}
for fname, url in zip(filenames, download_urls):
urls[fname] = url
return urls

Xet Storage Details

Size:
4.29 kB
·
Xet hash:
1cf356ceaa1e0bc892b4d2ef933391c55c6911d0b859dacf2442694f08d633d4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.