Buckets:

download
raw
2.21 kB
"""Functionality for working with datasets."""
from __future__ import annotations
import base64
import io
from typing import Literal
import calkit
import calkit.config
DEFAULT_ENGINE = calkit.config.read().dataframe_engine
def _get_df_lib(engine: str):
if engine == "pandas":
import pandas
return pandas
elif engine == "polars":
import polars
return polars
else:
raise ValueError("Unknown engine")
def list_datasets() -> list[dict]:
"""Read the Calkit metadata file and list out our datasets."""
ck_info = calkit.load_calkit_info()
return ck_info.get("datasets", [])
def read_dataset(
path: str,
engine: Literal["pandas", "polars"] = DEFAULT_ENGINE,
):
"""Read a dataset from a path.
Path can include the project owner/name like
someone/some-project:my-data-folder/data.csv
When a project is set via the ``CALKIT_PROJECT`` environmental variable,
we will use the API to fetch the data.
"""
def load_from_fobj(fobj, path: str):
"""Read from a filelike object or path."""
if path.endswith(".csv"):
return _get_df_lib(engine).read_csv(fobj)
elif path.endswith(".parquet"):
return _get_df_lib(engine).read_parquet(fobj)
project, path = calkit.project_and_path_from_path(path)
if project is not None:
if len(project.split("/")) != 2:
raise ValueError("Invalid project identifier (too many slashes)")
resp = calkit.hub.get(f"/projects/{project}/contents/{path}")
# If the response has a content key, that is a base64 encoded string
if (content := resp.get("content")) is not None:
# Load the content appropriately
content_bytes = base64.b64decode(content)
return load_from_fobj(io.BytesIO(content_bytes), path=path)
# If the response has a URL, we can fetch from that directly
elif (url := resp.get("url")) is not None:
return load_from_fobj(url, path=path)
else:
raise ValueError("No content or URL returned from API")
# Project is None, so let's just read a local file
return load_from_fobj(path, path)

Xet Storage Details

Size:
2.21 kB
·
Xet hash:
c2a7020645982fa1f5257b5187024e52fd392f7eab17b2e4cdb0a8de1d10b107

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.