owensong's picture
Launch complete Breeze TTS 2 ZeroGPU studio
ecfc07b verified
Raw History Blame Contribute Delete
5.41 kB
from __future__ import annotations
import logging
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
from .kv_cache import StaticShiftKVCache
logger = logging.getLogger(__name__)
T = TypeVar("T")
@dataclass
class ConvStateBlock:
cache_buf: torch.Tensor
left_cache_len: int
@dataclass
class TransConvStateBlock:
cache_buf: torch.Tensor
left_cache_len: int
@dataclass
class RequestStateSlot:
req_id: str | None
next_step: int
tail_flushed: bool
conv1d: dict[str, ConvStateBlock]
tconv1d: dict[str, TransConvStateBlock]
kv: StaticShiftKVCache
def reset_request_state(slot: RequestStateSlot) -> None:
slot.req_id = None
slot.next_step = 0
slot.tail_flushed = False
for block in slot.conv1d.values():
block.cache_buf.zero_()
for block in slot.tconv1d.values():
block.cache_buf.zero_()
for layer in slot.kv.layers:
layer.k.zero_()
layer.v.zero_()
class RequestStatePool(Generic[T]):
def __init__(
self,
create_slot: Callable[[], T],
reset_slot: Callable[[T], None],
max_active_reqs: int | None = None,
on_evict: Callable[[str], None] | None = None,
):
if max_active_reqs is not None and max_active_reqs <= 0:
raise ValueError(
f"max_active_reqs must be > 0 or None, got {max_active_reqs}"
)
self._create_slot = create_slot
self._reset_slot = reset_slot
self.max_active_reqs = max_active_reqs
self._on_evict = on_evict
self._req_to_slot: OrderedDict[str, int] = OrderedDict()
self._slot_to_req: list[str | None] = []
self._slots: list[T] = []
self._pinned: set[str] = set()
self._dynamic: dict[str, T] = {}
if self.max_active_reqs is not None:
self._slots = [self._create_slot() for _ in range(self.max_active_reqs)]
self._slot_to_req = [None for _ in range(self.max_active_reqs)]
def __len__(self) -> int:
return (
len(self._req_to_slot)
if self.max_active_reqs is not None
else len(self._dynamic)
)
def active_req_ids(self) -> list[str]:
if self.max_active_reqs is None:
return list(self._dynamic.keys())
return list(self._req_to_slot.keys())
def num_free_slots(self) -> int | None:
if self.max_active_reqs is None:
return None
return sum(1 for req_id in self._slot_to_req if req_id is None)
def pin(self, req_id: str) -> None:
self._pinned.add(req_id)
def unpin(self, req_id: str) -> None:
self._pinned.discard(req_id)
def _find_free_slot(self) -> int | None:
for idx, req_id in enumerate(self._slot_to_req):
if req_id is None:
return idx
return None
def _evict_lru(self) -> int:
victim_req_id: str | None = None
for req_id in self._req_to_slot.keys():
if req_id not in self._pinned:
victim_req_id = req_id
break
if victim_req_id is None:
logger.error(
"RequestStatePool found no evictable request. max_active_reqs=%s active_req_ids=%s pinned=%s",
self.max_active_reqs,
list(self._req_to_slot.keys()),
sorted(self._pinned),
)
raise RuntimeError("No evictable request state slot available.")
slot_idx = self._req_to_slot.pop(victim_req_id)
self._slot_to_req[slot_idx] = None
self._reset_slot(self._slots[slot_idx])
if self._on_evict is not None:
self._on_evict(victim_req_id)
logger.info(
"Evicted request state due to LRU capacity limit: req_id=%s slot_idx=%d",
victim_req_id,
slot_idx,
)
return slot_idx
def get(self, req_id: str) -> T:
if self.max_active_reqs is None:
value = self._dynamic.get(req_id)
if value is None:
value = self._create_slot()
self._dynamic[req_id] = value
return value
slot_idx = self._req_to_slot.get(req_id)
if slot_idx is None:
slot_idx = self._find_free_slot()
if slot_idx is None:
slot_idx = self._evict_lru()
self._slot_to_req[slot_idx] = req_id
self._req_to_slot[req_id] = slot_idx
self._reset_slot(self._slots[slot_idx])
self._req_to_slot.move_to_end(req_id)
return self._slots[slot_idx]
def pop(self, req_id: str) -> None:
self._pinned.discard(req_id)
if self.max_active_reqs is None:
self._dynamic.pop(req_id, None)
return
slot_idx = self._req_to_slot.pop(req_id, None)
if slot_idx is None:
return
self._reset_slot(self._slots[slot_idx])
self._slot_to_req[slot_idx] = None
def get_slot_index(self, req_id: str) -> int | None:
if self.max_active_reqs is None:
return None
return self._req_to_slot.get(req_id)