Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /ciris_sdk /pagination.py
| """ | |
| Pagination utilities for CIRIS SDK. | |
| Provides standardized cursor-based pagination for all list endpoints. | |
| """ | |
| import base64 | |
| import json | |
| import logging | |
| from typing import Any, AsyncIterator, Awaitable, Callable, Dict, Generic, List, Optional, TypeVar | |
| from pydantic import BaseModel, ConfigDict, Field | |
| logger = logging.getLogger(__name__) | |
| T = TypeVar("T", bound=BaseModel) | |
| class CursorInfo(BaseModel): | |
| """Decoded cursor information.""" | |
| offset: int = Field(0, description="Current offset") | |
| filters: Dict[str, Any] = Field(default_factory=dict, description="Applied filters") | |
| sort_key: Optional[str] = Field(None, description="Sort field") | |
| sort_value: Optional[Any] = Field(None, description="Last item's sort value") | |
| class PaginatedResponse(BaseModel, Generic[T]): | |
| """ | |
| Standardized paginated response. | |
| All list endpoints return data in this format with cursor-based pagination. | |
| """ | |
| items: List[T] = Field(..., description="List of items in current page") | |
| total: Optional[int] = Field(None, description="Total count if available") | |
| cursor: Optional[str] = Field(None, description="Cursor for next page") | |
| has_more: bool = Field(..., description="Whether more items exist") | |
| model_config = ConfigDict(arbitrary_types_allowed=True) | |
| class PageIterator(Generic[T]): | |
| """ | |
| Iterator for paginated results. | |
| Automatically fetches pages as needed for seamless iteration. | |
| """ | |
| def __init__( | |
| self, | |
| fetch_func: Callable[..., Awaitable[Any]], | |
| initial_params: Dict[str, Any], | |
| item_class: type[T], | |
| ): | |
| """ | |
| Initialize page iterator. | |
| Args: | |
| fetch_func: Async function to fetch a page | |
| initial_params: Initial query parameters | |
| item_class: Class for items in the response | |
| """ | |
| self.fetch_func = fetch_func | |
| self.params = initial_params.copy() | |
| self.item_class = item_class | |
| self._current_page: Optional[PaginatedResponse[T]] = None | |
| self._current_index = 0 | |
| self._exhausted = False | |
| async def __aiter__(self) -> AsyncIterator[T]: | |
| """Async iteration support.""" | |
| while not self._exhausted: | |
| # Fetch next page if needed | |
| if self._current_page is None or self._current_index >= len(self._current_page.items): | |
| await self._fetch_next_page() | |
| # Check if we're done | |
| if self._current_page is None or not self._current_page.items: | |
| self._exhausted = True | |
| break | |
| self._current_index = 0 | |
| # Yield current item | |
| yield self._current_page.items[self._current_index] | |
| self._current_index += 1 | |
| async def _fetch_next_page(self) -> None: | |
| """Fetch the next page of results.""" | |
| try: | |
| # Use cursor if we have one | |
| if self._current_page and self._current_page.cursor: | |
| self.params["cursor"] = self._current_page.cursor | |
| # Fetch page | |
| response = await self.fetch_func(**self.params) | |
| # Parse response | |
| if isinstance(response, dict): | |
| # Type assertion: we know item_class is bound to T | |
| assert isinstance(self.item_class, type) | |
| self._current_page = PaginatedResponse[T](**response) | |
| else: | |
| self._current_page = response | |
| # Check if we should continue | |
| if self._current_page is None or not self._current_page.has_more: | |
| self._exhausted = True | |
| except Exception as e: | |
| logger.error(f"Error fetching page: {e}") | |
| self._exhausted = True | |
| raise | |
| def encode_cursor(cursor_info: CursorInfo) -> str: | |
| """ | |
| Encode cursor information. | |
| Args: | |
| cursor_info: Cursor information to encode | |
| Returns: | |
| Base64-encoded cursor string | |
| """ | |
| data = cursor_info.model_dump(exclude_none=True) | |
| json_str = json.dumps(data, sort_keys=True) | |
| return base64.urlsafe_b64encode(json_str.encode()).decode() | |
| def decode_cursor(cursor: str) -> CursorInfo: | |
| """ | |
| Decode cursor string. | |
| Args: | |
| cursor: Base64-encoded cursor | |
| Returns: | |
| Decoded cursor information | |
| """ | |
| try: | |
| json_str = base64.urlsafe_b64decode(cursor.encode()).decode() | |
| data = json.loads(json_str) | |
| return CursorInfo(**data) | |
| except Exception as e: | |
| logger.error(f"Invalid cursor: {e}") | |
| # Return default cursor on error | |
| return CursorInfo() | |
| class PaginationParams(BaseModel): | |
| """Standard pagination parameters.""" | |
| cursor: Optional[str] = Field(None, description="Cursor for pagination") | |
| limit: int = Field(50, ge=1, le=200, description="Items per page") | |
| def to_dict(self) -> Dict[str, Any]: | |
| """Convert to dict for API calls.""" | |
| return self.model_dump(exclude_none=True) | |
| class QueryParams(PaginationParams): | |
| """ | |
| Extended query parameters with filtering. | |
| Base class for resource-specific query parameters. | |
| """ | |
| # Common filter fields | |
| since: Optional[str] = Field(None, description="Filter by creation time") | |
| until: Optional[str] = Field(None, description="Filter by creation time") | |
| sort: Optional[str] = Field(None, description="Sort field") | |
| order: Optional[str] = Field("desc", pattern="^(asc|desc)$", description="Sort order") | |
| # Search | |
| q: Optional[str] = Field(None, description="Search query") | |
| def to_dict(self) -> Dict[str, Any]: | |
| """Convert to dict, handling special cases.""" | |
| data = super().to_dict() | |
| # Remove empty strings | |
| return {k: v for k, v in data.items() if v != ""} | |
| async def paginate_all( | |
| fetch_func: Callable[..., Awaitable[Any]], | |
| params: Dict[str, Any], | |
| item_class: type[T], | |
| max_items: Optional[int] = None, | |
| ) -> List[T]: | |
| """ | |
| Fetch all pages of a paginated endpoint. | |
| Args: | |
| fetch_func: Async function to fetch a page | |
| params: Query parameters | |
| item_class: Class for items | |
| max_items: Maximum items to fetch (None = all) | |
| Returns: | |
| List of all items | |
| """ | |
| all_items = [] | |
| iterator = PageIterator(fetch_func, params, item_class) | |
| async for item in iterator: | |
| all_items.append(item) | |
| if max_items and len(all_items) >= max_items: | |
| break | |
| return all_items | |
Xet Storage Details
- Size:
- 6.48 kB
- Xet hash:
- 5465210c073bd238490f6e43577af97224b37d6c963ab5c44a659360a1ebc63d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.