File size: 5,951 Bytes
bf9e111 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | # -*- coding: utf-8 -*-
# Kiro Gateway
# https://github.com/jwadow/kiro-gateway
# Copyright (C) 2025 Jwadow
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Model metadata cache for Kiro Gateway.
Thread-safe storage for available model information
with TTL and lazy loading support.
"""
import asyncio
import time
from typing import Any, Dict, List, Optional
from loguru import logger
from kiro.config import MODEL_CACHE_TTL, DEFAULT_MAX_INPUT_TOKENS
class ModelInfoCache:
"""
Thread-safe cache for storing model metadata.
Uses Lazy Loading for population - data is loaded
only on first access or when cache is stale.
Attributes:
cache_ttl: Cache time-to-live in seconds
Example:
>>> cache = ModelInfoCache()
>>> await cache.update([{"modelId": "claude-sonnet-4", "tokenLimits": {...}}])
>>> info = cache.get("claude-sonnet-4")
>>> max_tokens = cache.get_max_input_tokens("claude-sonnet-4")
"""
def __init__(self, cache_ttl: int = MODEL_CACHE_TTL):
"""
Initializes the model cache.
Args:
cache_ttl: Cache time-to-live in seconds (default from config)
"""
self._cache: Dict[str, Dict[str, Any]] = {}
self._lock = asyncio.Lock()
self._last_update: Optional[float] = None
self._cache_ttl = cache_ttl
async def update(self, models_data: List[Dict[str, Any]]) -> None:
"""
Updates the model cache.
Thread-safely replaces cache contents with new data.
Args:
models_data: List of dictionaries with model information.
Each dictionary must contain the "modelId" key.
"""
async with self._lock:
logger.info(f"Updating model cache. Found {len(models_data)} models.")
self._cache = {model["modelId"]: model for model in models_data}
self._last_update = time.time()
def get(self, model_id: str) -> Optional[Dict[str, Any]]:
"""
Returns model information.
Args:
model_id: Model ID
Returns:
Dictionary with model information or None if model not found
"""
return self._cache.get(model_id)
def is_valid_model(self, model_id: str) -> bool:
"""
Check if model exists in dynamic cache.
Used by ModelResolver to verify if a model is available.
Args:
model_id: Model ID to check
Returns:
True if model exists in cache, False otherwise
"""
return model_id in self._cache
def add_hidden_model(self, display_name: str, internal_id: str) -> None:
"""
Add a hidden model to the cache.
Hidden models are not returned by Kiro /ListAvailableModels API
but are still functional. They are added to the cache so they
appear in our /v1/models endpoint.
Args:
display_name: Model name to display (e.g., "claude-3.7-sonnet")
internal_id: Internal Kiro ID (e.g., "CLAUDE_3_7_SONNET_20250219_V1_0")
"""
if display_name not in self._cache:
self._cache[display_name] = {
"modelId": display_name,
"modelName": display_name,
"description": f"Hidden model (internal: {internal_id})",
"tokenLimits": {"maxInputTokens": DEFAULT_MAX_INPUT_TOKENS},
"_internal_id": internal_id, # Store internal ID for reference
"_is_hidden": True, # Mark as hidden model
}
logger.debug(f"Added hidden model: {display_name} → {internal_id}")
def get_max_input_tokens(self, model_id: str) -> int:
"""
Returns maxInputTokens for the model.
Args:
model_id: Model ID
Returns:
Maximum number of input tokens or DEFAULT_MAX_INPUT_TOKENS
"""
model = self._cache.get(model_id)
if model and model.get("tokenLimits"):
return model["tokenLimits"].get("maxInputTokens") or DEFAULT_MAX_INPUT_TOKENS
return DEFAULT_MAX_INPUT_TOKENS
def is_empty(self) -> bool:
"""
Checks if the cache is empty.
Returns:
True if cache is empty
"""
return not self._cache
def is_stale(self) -> bool:
"""
Checks if the cache is stale.
Returns:
True if cache is stale (more than cache_ttl seconds have passed)
or if cache was never updated
"""
if not self._last_update:
return True
return time.time() - self._last_update > self._cache_ttl
def get_all_model_ids(self) -> List[str]:
"""
Returns a list of all model IDs in the cache.
Returns:
List of model IDs
"""
return list(self._cache.keys())
@property
def size(self) -> int:
"""Number of models in the cache."""
return len(self._cache)
@property
def last_update_time(self) -> Optional[float]:
"""Last update time (timestamp) or None."""
return self._last_update |