Buckets:

download
raw
45.5 kB
"""A filesystem-like object that follows ``fsspec`` and interacts with Calkit object storage to unify operations between private and public storage. The basic operation is as follows: 1. Make a request to the Calkit API over HTTP to get file operation info. 2. The API returns access details based on the configured storage backend. 3. Use the access details (presigned URL, OAuth, etc.) to perform the operation. Path format: ck://owner/project/path/to/file - owner: Calkit username/organization - project: Project name - path/to/file: File path within the project (optional) Supported storage backends (via the Calkit hub API): - Google Cloud Storage (GCS) - presigned URLs - Amazon S3 - presigned URLs - Google Drive - OAuth + API - Box - OAuth + API - Other storage providers as configured in the hub Multi-backend support: By default, the filesystem routes to the Calkit hub API endpoint of the active hub (CALKIT_HUB, the project's declared hub, or default_hub). To use a different Calkit hub: - DVC config: dvc remote modify myremote endpointurl https://api.other.com - URI query: ck://owner/project/file?endpoint_url=https://api.other.com Examples: # DVC remote (DVC will organize by MD5 hashes automatically) ck://owner/project # Subdirectory for organization ck://owner/project/data # Direct filesystem usage >>> from calkit.fs import CalkitFileSystem >>> fs = CalkitFileSystem() >>> with fs.open("ck://owner/project/file.txt", "rb") as f: ... content = f.read() # Using with pandas (automatic fsspec integration) >>> import pandas as pd >>> import calkit # Import to register the filesystem >>> df = pd.read_parquet("ck://owner/project/data.parquet") >>> df.to_parquet("ck://owner/project/output.parquet") # Using with polars (requires explicit fsspec.open) >>> import polars as pl >>> import fsspec >>> import calkit # Import to register the filesystem >>> # Reading >>> with fsspec.open("ck://owner/project/data.parquet", "rb") as f: ... df = pl.read_parquet(f) >>> # Writing >>> with fsspec.open("ck://owner/project/output.parquet", "wb") as f: ... df.write_parquet(f) Note: Pandas automatically uses fsspec for custom protocols, but Polars uses Rust's object_store crate which doesn't support fsspec. For Polars, use fsspec.open() explicitly to get a file-like object. The design supports transparent interaction with multiple storage backends or providers through a unified API. """ from __future__ import annotations import base64 import io import os import time import xml.etree.ElementTree as ET from typing import Any from urllib.parse import urlparse import requests from fsspec import AbstractFileSystem from fsspec.callbacks import DEFAULT_CALLBACK from fsspec.spec import AbstractBufferedFile from fsspec.utils import stringify_path from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError, Timeout import calkit def register_filesystem(): """Register the Calkit filesystem with fsspec's registry. This function is called automatically when the module is imported, but can also be called manually if needed. """ from fsspec.registry import register_implementation register_implementation("ck", "calkit.fs.CalkitFileSystem") # Register the filesystem when the module is imported register_filesystem() def _parse_path(path: str) -> tuple[str, str, str]: """Parse a Calkit path into (owner, project, file_path).""" path = stringify_path(path) parsed = urlparse(path) if parsed.scheme == "ck": # Standard format: ck://owner/project/file # netloc is the owner, path contains /project/file if parsed.netloc: raw_path = f"{parsed.netloc}{parsed.path}" else: raw_path = parsed.path.lstrip("/") else: # fsspec may pass protocol-stripped paths raw_path = path.lstrip("/") path_parts = [part for part in raw_path.split("/") if part] # Need at least owner/project if len(path_parts) < 2: raise ValueError( f"Invalid path format: {path}; Expected ck://owner/project/path" ) owner = path_parts[0] project = path_parts[1] file_path = "/".join(path_parts[2:]) if len(path_parts) > 2 else "" return owner, project, file_path class CalkitFileSystem(AbstractFileSystem): """An fsspec-compatible filesystem for Calkit hub storage. This filesystem makes requests to the Calkit API to get file operation information, then uses the appropriate method to interact with the underlying storage backend. The Calkit hub API acts as a compatibility layer that: - Determines which storage backend is configured for each project - Returns appropriate access credentials (presigned URLs, OAuth tokens, etc.) - Supports GCS, S3, Google Drive, Box, Azure, and other providers Path format: ck://owner/project/path/to/file Users can organize their files however they want. For example: - DVC will automatically organize by MD5 hashes - Users can create subdirectories for organization (data/, models/, etc.) - Files can be stored at the project root Endpoints are configured via: - endpointurl parameter (for DVC remotes): Route to different Calkit instances - endpoint_url query parameter (for URIs): Ad-hoc endpoint specification - the active hub: CALKIT_HUB, the working directory project's declared ``hub``, or the ``default_hub`` config value - Defaults to calkit.io when unspecified This design allows for: - Multiple storage backend support without client-side changes - Multiple storage backends with configurable endpoints - Future protocol upgrades (e.g., SFTP) without breaking API compatibility - Unified interface regardless of underlying storage provider Attributes ---------- protocol : str The protocol scheme for this filesystem ("ck") base_url : str The Calkit hub API endpoint URL, configured via endpointurl in DVC config, endpoint_url in URI query, or the active hub. Defaults to production (https://api.calkit.io) when unspecified. """ protocol = "ck" def __init__(self, *args, **kwargs): """Initialize the filesystem.""" super().__init__(*args, **kwargs) self._session = requests.Session() self._base_url = kwargs.get("endpoint_url") @property def base_url(self) -> str: """Get the base URL for the Calkit hub API.""" return self._base_url or calkit.hub.get_base_url() @staticmethod def _normalize_info( file_path: str, result: dict[str, Any] ) -> dict[str, Any]: return { "name": result.get("name", file_path), "size": result.get("size", 0), "type": result.get("type", "file"), "time_modified": result.get("time_modified"), } def _get_fs_op_info( self, owner: str, project: str, path: str, operation: str = "get", content_length: int | None = None, content_type: str | None = None, detail: bool = False, ) -> dict: """Get file operation info from the Calkit API.""" endpoint = f"/projects/{owner}/{project}/fs/ops" request_body: dict[str, Any] = { "operation": operation, "path": path, "detail": detail, } if content_length is not None: request_body["content_length"] = content_length if content_type is not None: request_body["content_type"] = content_type resp = calkit.hub.post( endpoint, json=request_body, base_url=self.base_url ) # Validate response has required fields if "backend" not in resp: raise ValueError( f"Invalid API response: {resp}; Expected 'backend' field" ) return resp def _get_fs_op_info_batch( self, owner: str, project: str, paths: list[str], operation: str, include: list[str] | None = None, detail: bool = False, ) -> dict: """Get batch file operation info from the Calkit API.""" endpoint = f"/projects/{owner}/{project}/fs/ops/batch" request_body: dict[str, Any] = { "operation": operation, "detail": detail, "paths": paths, } if include: request_body["include"] = include resp = calkit.hub.post( endpoint, json=request_body, base_url=self.base_url ) if "backend" not in resp: raise ValueError( f"Invalid API response: {resp}; Expected 'backend' field" ) return resp def _get_info_for_parsed_path( self, owner: str, project: str, file_path: str ) -> dict[str, Any]: try: operation_info = self._get_fs_op_info( owner, project, file_path, operation="info" ) except HTTPError as e: # If the API returns 404, convert to FileNotFoundError if "404" in str(e): raise FileNotFoundError(f"ck://{owner}/{project}/{file_path}") raise if "result" in operation_info: info = self._normalize_info(file_path, operation_info["result"]) else: resp = self._execute_operation(operation_info, "info") if resp.status_code == 404: raise FileNotFoundError(f"ck://{owner}/{project}/{file_path}") resp.raise_for_status() info = self._normalize_info(file_path, resp.json()) return info def _get_exists_for_parsed_path( self, owner: str, project: str, file_path: str ) -> bool: operation_info = self._get_fs_op_info( owner, project, file_path, operation="exists" ) if "result" in operation_info: exists = bool(operation_info["result"].get("exists", False)) else: resp = self._execute_operation(operation_info, "exists") resp.raise_for_status() exists = bool(resp.json().get("exists", False)) return exists def _execute_operation( self, operation_info: dict[str, Any], operation: str, data: bytes | None = None, file_obj: Any = None, file_size: int | None = None, headers: dict | None = None, callback=DEFAULT_CALLBACK, ) -> requests.Response: """Execute a file operation using the provided operation info. Exactly one of ``data`` or ``file_obj`` should be provided for put operations. When ``file_obj`` is given it must be a seekable binary file-like object and ``file_size`` must be set; part/chunk bodies are streamed from it instead of being held in memory, which keeps memory bounded for large uploads. """ if data is not None and file_obj is not None: raise ValueError("Provide data or file_obj, not both") if file_obj is not None: if not callable(getattr(file_obj, "seek", None)): raise TypeError( "file_obj must be a seekable binary file-like object" ) if file_size is None: raise ValueError( "file_size must be provided when file_obj is given" ) # Total payload size for progress reporting. payload_size = len(data) if data is not None else (file_size or 0) # Retries are intentionally narrow: transient transport errors and # transient 5xx responses from the storage backend (e.g. GCS/S3 # returning 503 under load). Non-5xx statuses (308 resume acks, 404, # etc.) are returned to the caller untouched. max_retries = 2 retry_backoff = 1.0 transient_statuses = (500, 502, 503, 504) def _request_with_retry(func): delay = retry_backoff for attempt in range(max_retries + 1): try: resp = func() except (Timeout, RequestsConnectionError): if attempt >= max_retries: raise else: status = getattr(resp, "status_code", None) if ( status not in transient_statuses or attempt >= max_retries ): return resp if delay > 0: time.sleep(delay) delay *= 2 def _prepare_stream(request_headers: dict) -> Any: """Return the request body, setting Content-Length if streaming.""" if data is not None: return data if file_obj is None: return None file_obj.seek(0) # Explicit Content-Length avoids chunked transfer-encoding, which # presigned URL signatures typically reject. request_headers.setdefault("Content-Length", str(file_size)) return file_obj # Extract access info from the operation info access = operation_info.get("access") if not access: raise ValueError("Missing 'access' field in operation info") kind = access.get("kind") if not kind: raise ValueError("Missing 'kind' field in access info") # Handle different access types if kind == "presigned-url": # Simple presigned URL - just make the HTTP request url = access.get("url") if not url: raise ValueError("Missing 'url' field for presigned-url") http_method = access.get("http_method") or ( "PUT" if operation == "put" else "GET" ) request_headers = dict(access.get("headers") or {}) if headers: request_headers.update(headers) params = access.get("params") resp = _request_with_retry( lambda: self._session.request( method=http_method.upper(), url=url, headers=request_headers, params=params, data=_prepare_stream(request_headers), timeout=120, ) ) if operation == "put" and payload_size: callback.relative_update(payload_size) return resp elif kind == "http-request": # Generic HTTP request with custom headers url = access.get("url") if not url: raise ValueError("Missing 'url' field for http-request") http_method = access.get("http_method") or ( "PUT" if operation == "put" else "GET" ) request_headers = dict(access.get("headers") or {}) if headers: request_headers.update(headers) params = access.get("params") resp = _request_with_retry( lambda: self._session.request( method=http_method.upper(), url=url, headers=request_headers, params=params, data=_prepare_stream(request_headers), timeout=120, ) ) if operation == "put" and payload_size: callback.relative_update(payload_size) return resp elif kind == "presigned-multipart": # S3 multipart upload using server-provided part URLs if data is None and file_obj is None: raise ValueError("Data required for multipart upload") upload_id = access.get("upload_id") part_urls = access.get("part_urls") complete_url = access.get("complete_url") part_size = access.get("part_size_bytes") if not upload_id: raise ValueError( "Missing 'upload_id' field for presigned-multipart" ) if not isinstance(part_urls, list) or len(part_urls) == 0: raise ValueError( "Missing or invalid 'part_urls' for presigned-multipart" ) if not complete_url: raise ValueError( "Missing 'complete_url' for presigned-multipart" ) if not isinstance(part_size, int) or part_size <= 0: raise ValueError( "Missing or invalid 'part_size_bytes' for " "presigned-multipart" ) content_type = access.get("content_type") total_bytes = len(data) if data is not None else int(file_size) total_parts_needed = (total_bytes + part_size - 1) // part_size if total_parts_needed > len(part_urls): raise ValueError( "Insufficient part URLs for multipart upload " f"(need {total_parts_needed}, got {len(part_urls)})" ) uploaded_parts: list[tuple[int, str]] = [] for part_num in range(1, total_parts_needed + 1): start = (part_num - 1) * part_size end = min(start + part_size, total_bytes) if data is not None: part_data = data[start:end] else: file_obj.seek(start) part_data = file_obj.read(end - start) part_url = part_urls[part_num - 1] part_headers = {} if content_type: part_headers["Content-Type"] = str(content_type) part_resp = _request_with_retry( lambda: self._session.put( part_url, headers=part_headers, data=part_data, timeout=120, ) ) part_resp.raise_for_status() etag = part_resp.headers.get("ETag") if not etag: raise ValueError( f"Missing ETag for uploaded multipart part {part_num}" ) uploaded_parts.append((part_num, etag.strip())) callback.relative_update(len(part_data)) complete_root = ET.Element("CompleteMultipartUpload") for part_num, etag in uploaded_parts: part_el = ET.SubElement(complete_root, "Part") ET.SubElement(part_el, "PartNumber").text = str(part_num) ET.SubElement(part_el, "ETag").text = etag complete_body = ET.tostring( complete_root, encoding="utf-8", xml_declaration=True ) complete_resp = _request_with_retry( lambda: self._session.post( complete_url, headers={"Content-Type": "application/xml"}, data=complete_body, timeout=120, ) ) complete_resp.raise_for_status() return complete_resp elif kind == "presigned-chunked": # GCS resumable upload - requires multiple requests if data is None and file_obj is None: raise ValueError("Data required for chunked upload") init_url = access.get("init_url") if not init_url: raise ValueError( "Missing 'init_url' field for presigned-chunked" ) chunk_size = access.get("chunk_size_bytes", 5 * 1024 * 1024) if not isinstance(chunk_size, int) or chunk_size <= 0: raise ValueError( "Missing or invalid 'chunk_size_bytes' for " "presigned-chunked" ) content_type = access.get("content_type") init_method = access.get("http_method", "POST") init_params = access.get("params") chunk_upload_method = access.get("chunk_http_method", "PUT") # Step 1: Initiate resumable upload init_headers = dict(access.get("headers") or {}) if content_type: init_headers["Content-Type"] = str(content_type) init_headers["Content-Length"] = "0" # Init request has no body init_resp = _request_with_retry( lambda: self._session.request( method=init_method.upper(), url=init_url, headers=init_headers, params=init_params, timeout=30, ) ) init_resp.raise_for_status() # Get the session URI from the Location header session_uri = init_resp.headers.get("Location") if not session_uri: raise ValueError( "Failed to get session URI from resumable upload init " "response; " "Expected 'Location' header" ) def _cancel_resumable_upload() -> None: delete_fn = getattr(self._session, "delete", None) if callable(delete_fn): try: delete_fn(session_uri, timeout=30) except Exception: pass # Step 2: Upload data in chunks total_size = len(data) if data is not None else int(file_size) offset = 0 chunk_headers_base = dict( access.get("chunk_headers") or access.get("headers") or {} ) # x-goog-resumable is only required to initiate the session. if "chunk_headers" not in access: for header_key in list(chunk_headers_base.keys()): if header_key.lower() == "x-goog-resumable": chunk_headers_base.pop(header_key) def _parse_next_offset_from_range( range_header: str | None, ) -> int | None: if not range_header: return None value = range_header.strip() if "=" in value: value = value.split("=", 1)[1].strip() if "-" not in value: return None start_str, end_str = value.split("-", 1) if not start_str.isdigit() or not end_str.isdigit(): return None start = int(start_str) end = int(end_str) if start < 0 or end < start: return None return end + 1 try: while offset < total_size: chunk_end = min(offset + chunk_size, total_size) if data is not None: chunk_data = data[offset:chunk_end] else: file_obj.seek(offset) chunk_data = file_obj.read(chunk_end - offset) # Set Content-Range header for the chunk chunk_headers = { **chunk_headers_base, "Content-Length": str(len(chunk_data)), "Content-Range": ( f"bytes {offset}-{chunk_end - 1}/{total_size}" ), } chunk_resp = _request_with_retry( lambda: self._session.request( method=chunk_upload_method.upper(), url=session_uri, headers=chunk_headers, params=access.get("chunk_params"), data=chunk_data, timeout=120, ) ) # Check response if chunk_resp.status_code == 308: # Resume incomplete: trust server ack instead of local # send ack_range = chunk_resp.headers.get("Range") next_offset = _parse_next_offset_from_range(ack_range) if next_offset is None: raise ValueError( "Chunked upload received 308 without a valid " "Range acknowledgement" ) if next_offset <= offset: raise ValueError( "Chunked upload made no forward progress " f"(offset={offset}, ack={ack_range!r})" ) if next_offset > chunk_end: raise ValueError( "Chunked upload ack exceeds bytes sent in the " "current request " f"(sent_end={chunk_end}, ack={ack_range!r})" ) if next_offset >= total_size: raise ValueError( "Chunked upload acknowledged full payload with " "status 308 instead of final success" ) callback.relative_update(next_offset - offset) offset = next_offset elif chunk_resp.status_code in (200, 201): # Final success is only valid when sending the final # range if chunk_end != total_size: raise ValueError( "Chunked upload returned success before all " "bytes were sent" ) callback.relative_update(chunk_end - offset) return chunk_resp else: # Unexpected status chunk_resp.raise_for_status() raise ValueError( "Unexpected status code during chunked upload: " f"{chunk_resp.status_code}" ) except Exception: _cancel_resumable_upload() raise # If we reach here, the upload completed without getting a 200/201 # This shouldn't happen, but handle it gracefully raise ValueError( "Chunked upload completed but no success response received" ) elif kind == "sftp": # SFTP access raise NotImplementedError("SFTP access not yet implemented") else: raise ValueError(f"Unsupported access kind: {kind}") def _open( self, path: str, mode: str = "rb", block_size: int | None = None, autocommit: bool = True, cache_options: dict | None = None, **kwargs, ) -> CalkitFile: """Open a file for reading or writing.""" owner, project, file_path = _parse_path(path) return CalkitFile( self, path, mode, block_size, autocommit, owner=owner, project=project, file_path=file_path, cache_options=cache_options, **kwargs, ) def ls( self, path: str, detail: bool = False, refresh: bool = False, **kwargs ) -> list[str] | list[dict]: """List files in a directory.""" owner, project, file_path = _parse_path(path) # Get operation info from API operation_info = self._get_fs_op_info( owner, project, file_path, operation="list", detail=detail ) # Check if server provided the result directly if "result" in operation_info: paths = operation_info["result"].get("paths", []) else: # Server returned instructions; execute the operation resp = self._execute_operation(operation_info, "list") resp.raise_for_status() result = resp.json() paths = result.get("paths", []) # Ensure paths have the protocol-stripped format expected by fsspec # Format depends on detail flag: # - detail=False: list of strings (paths without protocol) # - detail=True: list of dicts with 'name', 'size', 'type' keys if detail: # If paths is already a list of dicts, return as-is # Otherwise, convert strings to minimal dict format if paths and isinstance(paths[0], dict): return paths else: # Convert list of strings to list of dicts return [ {"name": p, "size": None, "type": "file"} for p in paths ] else: # Return list of strings if paths and isinstance(paths[0], dict): return [p["name"] for p in paths] else: return paths def find( self, path, maxdepth=None, withdirs=False, detail=False, **kwargs ): """Recursively find all files under a path.""" owner, project, file_path = _parse_path(path) operation_info = self._get_fs_op_info( owner, project, file_path, operation="find", detail=detail ) # Check if server provided the result directly if "result" in operation_info: paths = operation_info["result"].get( "paths", [] if not detail else {} ) else: # Server returned instructions; execute the operation resp = self._execute_operation(operation_info, "find") # Handle 404: directory doesn't exist yet, return empty results if resp.status_code == 404: return [] if not detail else {} resp.raise_for_status() result = resp.json() paths = result.get("paths", [] if not detail else {}) # Normalize the paths format: # - detail=False: list of strings # - detail=True: dict mapping path -> info dict if detail: # Convert to dict format if it's a list if isinstance(paths, list): if paths and isinstance(paths[0], dict): # List of dicts - convert to dict mapping name -> info paths = {p["name"]: p for p in paths} else: # List of strings - convert to dict with minimal info paths = { p: {"name": p, "size": None, "type": "file"} for p in paths } else: # Convert to list format if it's a dict if isinstance(paths, dict): paths = list(paths.keys()) # Ensure it's a list of strings elif paths and isinstance(paths[0], dict): paths = [p["name"] for p in paths] # Apply maxdepth filtering if needed if maxdepth is not None: base_depth = file_path.count("/") if file_path else 0 if detail and isinstance(paths, dict): paths = { p: info for p, info in paths.items() if p.count("/") - base_depth <= maxdepth } elif detail: paths = {} else: paths = [ p for p in paths if p.count("/") - base_depth <= maxdepth ] # Filter out directories if not requested if not withdirs: if detail and isinstance(paths, dict): paths = { p: info for p, info in paths.items() if info.get("type") == "file" } else: # When detail=False, we don't have type info in the list # The backend should handle this by only returning files pass return paths def exists(self, path: str, **kwargs) -> bool: """Check if a path exists.""" owner, project, file_path = _parse_path(path) return self._get_exists_for_parsed_path(owner, project, file_path) def exists_many(self, paths: list[str], **kwargs) -> list[bool]: """Check existence for multiple paths in a batch API call.""" if not paths: return [] grouped: dict[tuple[str, str], list[tuple[int, str, str]]] = {} for idx, path in enumerate(paths): owner, project, file_path = _parse_path(path) grouped.setdefault((owner, project), []).append( (idx, path, file_path) ) results: list[bool | None] = [None] * len(paths) for (owner, project), entries in grouped.items(): file_paths = [file_path for _, _, file_path in entries] index_by_file_path = { file_path: [idx for idx, _, fp in entries if fp == file_path] for file_path in set([fp for _, _, fp in entries]) } try: resp = self._get_fs_op_info_batch( owner=owner, project=project, paths=file_paths, operation="exists", include=["exists"], ) batch_results = resp.get("results") if isinstance(batch_results, dict): for file_path, value in batch_results.items(): exists = bool( value.get("exists", False) if isinstance(value, dict) else value ) for idx in index_by_file_path.get(file_path, []): results[idx] = exists except Exception: pass # Fall back to single-path exists for any missing results for idx, _, file_path in entries: if results[idx] is None: try: results[idx] = self._get_exists_for_parsed_path( owner, project, file_path ) except Exception: results[idx] = False return [bool(v) for v in results] def info(self, path: str, **kwargs) -> dict: """Get file metadata (name, size, type).""" owner, project, file_path = _parse_path(path) return self._get_info_for_parsed_path(owner, project, file_path) def info_many( self, paths: list[str], **kwargs ) -> dict[str, dict[str, Any]]: """Get metadata for multiple paths in a batch API call.""" if not paths: return {} grouped: dict[tuple[str, str], list[tuple[str, str]]] = {} results: dict[str, dict[str, Any]] = {} for path in paths: owner, project, file_path = _parse_path(path) grouped.setdefault((owner, project), []).append((path, file_path)) for (owner, project), entries in grouped.items(): file_paths = [file_path for _, file_path in entries] paths_by_file_path = { file_path: [p for p, fp in entries if fp == file_path] for file_path in set([fp for _, fp in entries]) } try: resp = self._get_fs_op_info_batch( owner=owner, project=project, paths=file_paths, operation="info", include=["info", "content"], ) batch_results = resp.get("results") if isinstance(batch_results, dict): for file_path, value in batch_results.items(): if not isinstance(value, dict): continue # Batch API returns shape: # {"info": {...}, "content_base64": "...", ...} info_payload = value.get("info") if not isinstance(info_payload, dict): continue info = self._normalize_info(file_path, info_payload) content_b64 = value.get("content_base64") if isinstance(content_b64, str): try: info["content"] = base64.b64decode( content_b64, validate=True ) except Exception: pass for original_path in paths_by_file_path.get( file_path, [] ): results[original_path] = info except Exception: pass return results def cat_file( self, path: str, start: int | None = None, end: int | None = None, **kwargs, ) -> bytes: """Read file contents.""" owner, project, file_path = _parse_path(path) # Get file operation info from API (one API call per file) operation_info = self._get_fs_op_info( owner, project, file_path, operation="get" ) # Add Range header if reading a specific byte range headers = {} if start is not None and end is not None: headers["Range"] = f"bytes={start}-{end - 1}" elif start is not None: headers["Range"] = f"bytes={start}-" elif end is not None: headers["Range"] = f"bytes=0-{end - 1}" # Execute the get operation resp = self._execute_operation(operation_info, "get", headers=headers) resp.raise_for_status() return resp.content def rm_file(self, path: str, **kwargs): """Delete a file.""" owner, project, file_path = _parse_path(path) # Get file operation info from API operation_info = self._get_fs_op_info( owner, project, file_path, "delete" ) # Execute the delete operation resp = self._execute_operation(operation_info, "delete") resp.raise_for_status() def mv(self, path1: str, path2: str, **kwargs): """Move or rename a file (copy + delete).""" self.copy(path1, path2, **kwargs) self.rm_file(path1, **kwargs) def cp_file(self, path1: str, path2: str, **kwargs): """Copy a file.""" with self.open(path1, "rb") as src: data = src.read() # For binary write to a file, pass the bytes directly with self.open(path2, "wb") as dst: dst.write(data) # type: ignore[arg-type] def makedir(self, path, create_parents=True, **kwargs): """Create a directory (no-op for object storage). TODO: Handle other backends when they are supported. """ pass def makedirs(self, path, exist_ok=False): """Create directories (no-op for object storage). TODO: Handle other backends when they are supported. """ pass def put_file( self, lpath, rpath, callback=DEFAULT_CALLBACK, mode="overwrite", **kwargs, ): """Upload a local file to Calkit hub storage. Overrides the default fsspec implementation to report progress based on actual bytes uploaded rather than bytes written to the local buffer. """ if mode == "create" and self.exists(rpath): raise FileExistsError if os.path.isdir(lpath): self.makedirs(rpath, exist_ok=True) return None owner, project, file_path = _parse_path(rpath) size = os.path.getsize(lpath) callback.set_size(size) operation_info = self._get_fs_op_info( owner, project, file_path, "put", content_length=size, ) # Stream the file from disk rather than loading it into memory. This # keeps RAM bounded when DVC pushes many (or very large) files in # parallel via its worker pool. with open(lpath, "rb") as f: resp = self._execute_operation( operation_info, "put", file_obj=f, file_size=size, callback=callback, ) resp.raise_for_status() class CalkitFile(AbstractBufferedFile): """A file-like object for reading/writing from Calkit hub storage. This class handles buffering and delegates actual I/O to the underlying storage backend (GCS, S3, Google Drive, Box, etc.) via the Calkit API. Attributes ---------- owner : str Calkit owner/username project : str Calkit project name file_path : str Path within the project operation_info : dict | None Cached operation info from API (contains backend details) uploaded_bytes : int Total bytes uploaded (for tracking) """ def __init__( self, fs: CalkitFileSystem, path: str, mode: str = "rb", block_size: int | None = None, autocommit: bool = True, owner: str | None = None, project: str | None = None, file_path: str | None = None, cache_options: dict | None = None, **kwargs, ): self.owner = owner self.project = project self.file_path = file_path self.operation_info = None # Cached operation info from API self.uploaded_bytes = 0 # Track total bytes uploaded # fsspec expects block_size to be an integer, not None if block_size is None: block_size = 5 * 1024 * 1024 # Default 5MB super().__init__( fs, path, mode=mode, block_size=block_size, # type: ignore[arg-type] autocommit=autocommit, cache_options=cache_options, **kwargs, ) def _fetch_range(self, start: int, end: int) -> bytes: """Fetch a byte range from the file.""" # Fast path: use inline content from info/info_many when available. try: details = self.details content = ( details.get("content") if isinstance(details, dict) else None ) if isinstance(content, (bytes, bytearray)): return bytes(content)[start:end] except Exception: pass if self.operation_info is None: # Get file operation info from API self.operation_info = self.fs._get_fs_op_info( self.owner, self.project, self.file_path, "get" ) # Add Range header for partial content # For backends where range is unsupported, the Calkit hub API can # choose to ignore this header or return a backend-specific request # configuration headers = {"Range": f"bytes={start}-{end - 1}"} # Execute the get operation with range header resp = self.fs._execute_operation( self.operation_info, "get", headers=headers ) resp.raise_for_status() return resp.content def _upload_chunk(self, final: bool = False) -> int | bool: """Upload buffered data to object storage.""" if not final: # For non-final chunks, we don't upload yet (buffer accumulates) return False # Get the data to upload from the buffer if self.buffer is None: return 0 data = self.buffer.getvalue() ndata = len(data) if ndata == 0: return 0 if self.operation_info is None: raise RuntimeError( "Upload not initiated; Call _initiate_upload first" ) self.operation_info = self.fs._get_fs_op_info( self.owner, self.project, self.file_path, "put", content_length=ndata, ) # Execute the put operation resp = self.fs._execute_operation( self.operation_info, "put", data=data, ) resp.raise_for_status() self.uploaded_bytes += ndata # Clear the buffer after successful upload self.buffer = io.BytesIO() return ndata def _initiate_upload(self): """Get upload credentials from the Calkit API.""" self.operation_info = self.fs._get_fs_op_info( self.owner, self.project, self.file_path, "put" )
"""A filesystem-like object that follows ``fsspec`` and interacts with Calkit
object storage to unify operations between private and public storage.
The basic operation is as follows:
1. Make a request to the Calkit API over HTTP to get file operation info.
2. The API returns access details based on the configured storage backend.
3. Use the access details (presigned URL, OAuth, etc.) to perform the operation.
Path format:
ck://owner/project/path/to/file
- owner: Calkit username/organization
- project: Project name
- path/to/file: File path within the project (optional)
Supported storage backends (via the Calkit hub API):
- Google Cloud Storage (GCS) - presigned URLs
- Amazon S3 - presigned URLs
- Google Drive - OAuth + API
- Box - OAuth + API
- Other storage providers as configured in the hub
Multi-backend support:
By default, the filesystem routes to the Calkit hub API endpoint
of the active hub (CALKIT_HUB, the project's declared hub, or
default_hub). To use a different Calkit hub:
- DVC config: dvc remote modify myremote endpointurl https://api.other.com
- URI query: ck://owner/project/file?endpoint_url=https://api.other.com
Examples:
# DVC remote (DVC will organize by MD5 hashes automatically)
ck://owner/project
# Subdirectory for organization
ck://owner/project/data
# Direct filesystem usage
>>> from calkit.fs import CalkitFileSystem
>>> fs = CalkitFileSystem()
>>> with fs.open("ck://owner/project/file.txt", "rb") as f:
... content = f.read()
# Using with pandas (automatic fsspec integration)
>>> import pandas as pd
>>> import calkit # Import to register the filesystem
>>> df = pd.read_parquet("ck://owner/project/data.parquet")
>>> df.to_parquet("ck://owner/project/output.parquet")
# Using with polars (requires explicit fsspec.open)
>>> import polars as pl
>>> import fsspec
>>> import calkit # Import to register the filesystem
>>> # Reading
>>> with fsspec.open("ck://owner/project/data.parquet", "rb") as f:
... df = pl.read_parquet(f)
>>> # Writing
>>> with fsspec.open("ck://owner/project/output.parquet", "wb") as f:
... df.write_parquet(f)
Note:
Pandas automatically uses fsspec for custom protocols, but Polars uses
Rust's object_store crate which doesn't support fsspec. For Polars, use
fsspec.open() explicitly to get a file-like object.
The design supports transparent interaction with multiple storage backends
or providers through a unified API.
"""
from __future__ import annotations
import base64
import io
import os
import time
import xml.etree.ElementTree as ET
from typing import Any
from urllib.parse import urlparse
import requests
from fsspec import AbstractFileSystem
from fsspec.callbacks import DEFAULT_CALLBACK
from fsspec.spec import AbstractBufferedFile
from fsspec.utils import stringify_path
from requests.exceptions import ConnectionError as RequestsConnectionError
from requests.exceptions import HTTPError, Timeout
import calkit
def register_filesystem():
"""Register the Calkit filesystem with fsspec's registry.
This function is called automatically when the module is imported,
but can also be called manually if needed.
"""
from fsspec.registry import register_implementation
register_implementation("ck", "calkit.fs.CalkitFileSystem")

Xet Storage Details

Size:
45.5 kB
·
Xet hash:
03e1119c1b45e2773766db7b149d3c39b7b6e79ce23360d75b03a638b9e3edab

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.