Create app.py

app.py

@@ -0,0 +1,1389 @@
+import os
+import gc
+import json
+import random
+import torch
+import asyncio
+import logging
+import time
+import queue
+import sys
+from typing import List, Dict, Any, Optional, Union, AsyncGenerator, Tuple
+from fastapi import FastAPI, HTTPException, Query, Request, Depends, status
+from fastapi.responses import StreamingResponse, PlainTextResponse, HTMLResponse, JSONResponse
+from fastapi.security import APIKeyHeader
+from pydantic import BaseModel, Field, ValidationError, validator
+from transformers import (
+    AutoConfig, AutoModelForCausalLM, AutoTokenizer,
+    GenerationConfig, LogitsProcessorList,
+    MinLengthLogitsProcessor, MaxLengthCriteria,
+    StoppingCriteriaList, StoppingCriteria,
+    TextStreamer,
+)
+try:
+    from transformers.generation.utils import GenerationOutput
+except ImportError:
+    GenerationOutput = Any
+from concurrent.futures import ThreadPoolExecutor
+import uvicorn
+import math
+import torch.nn.functional as F
+import copy
+
+class SyncToAsyncQueue:
+    def __init__(self):
+        self._queue = queue.Queue()
+        try:
+            self._loop = asyncio.get_event_loop()
+        except RuntimeError:
+            self._loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._loop)
+        self._get_waiters = []
+        self._closed = False
+
+    def put_nowait(self, item):
+        if self._closed:
+             raise asyncio.QueueClosed("Queue is closed")
+        self._queue.put_nowait(item)
+        while self._get_waiters:
+             waiter = self._get_waiters.pop(0)
+             if not waiter.done():
+                 waiter.set_result(None)
+
+    async def get(self):
+        if self._closed and self._queue.empty():
+             raise asyncio.QueueClosed("Queue is closed")
+        if not self._queue.empty():
+             return self._queue.get_nowait()
+        waiter = self._loop.create_future()
+        self._get_waiters.append(waiter)
+        try:
+             await waiter
+        except asyncio.CancelledError:
+             if waiter in self._get_waiters:
+                 self._get_waiters.remove(waiter)
+             raise
+        if self._closed and self._queue.empty():
+             raise asyncio.QueueClosed("Queue is closed")
+        if not self._queue.empty():
+             return self._queue.get_nowait()
+        raise asyncio.QueueClosed("Queue is closed unexpectedly after waiter finished")
+
+
+    def close(self):
+         self._closed = True
+         while self._get_waiters:
+             waiter = self._get_waiters.pop(0)
+             if not waiter.done():
+                 waiter.set_exception(asyncio.QueueClosed("Queue is closed"))
+
+class QueueTextStreamer(TextStreamer):
+    def __init__(self, queue: SyncToAsyncQueue, tokenizer, skip_prompt: bool = False, **decode_kwargs):
+        class QueueWriter:
+            def __init__(self, queue):
+                self._queue = queue
+
+            def write(self, text):
+                if text:
+                     try:
+                         self._queue.put_nowait({"type": "token", "text": text})
+                     except asyncio.QueueClosed:
+                         pass
+
+            def flush(self):
+                 pass
+
+            def isatty(self): return False
+            def writable(self): return True
+
+        self.queue = queue
+        self.queue_writer = QueueWriter(queue)
+        super().__init__(tokenizer, self.queue_writer, skip_prompt=skip_prompt, **decode_kwargs)
+
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+         if text:
+             try:
+                 self.queue.put_nowait({"type": "token", "text": text})
+             except asyncio.QueueClosed:
+                 pass
+         if stream_end:
+             try:
+                  self.queue.put_nowait({"type": "stream_end_signal"})
+             except asyncio.QueueClosed:
+                  pass
+
+    def __call__(self, token_ids, scores=None, **kwargs):
+         super().__call__(token_ids, scores, **kwargs)
+
+class StopSequenceCriteria(StoppingCriteria):
+    def __init__(self, stop_sequences: List[str], tokenizer: AutoTokenizer):
+        self.tokenizer = tokenizer
+        self.stop_sequences_text = []
+        self.stop_sequence_ids = []
+        for seq in stop_sequences:
+            if seq:
+                encoded_ids = tokenizer.encode(seq, add_special_tokens=False)
+                decoded_text = tokenizer.decode(encoded_ids, skip_special_tokens=True)
+                if decoded_text:
+                    self.stop_sequences_text.append(decoded_text)
+                    self.stop_sequence_ids.append(encoded_ids)
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        if not self.stop_sequence_ids and not self.stop_sequences_text:
+            return False
+        input_ids_list = input_ids[0].tolist()
+        for stop_seq_ids in self.stop_sequence_ids:
+            stop_len = len(stop_seq_ids)
+            if len(input_ids_list) >= stop_len:
+                if input_ids_list[-stop_len:] == stop_seq_ids:
+                    return True
+        check_tail_len = max(len(ids) for ids in self.stop_sequence_ids) + 10 if self.stop_sequence_ids else 50
+        tail_ids = input_ids_list[-min(check_tail_len, len(input_ids_list)):]
+        tail_text = self.tokenizer.decode(tail_ids, skip_special_tokens=True)
+        for stop_seq_text in self.stop_sequences_text:
+            if stop_seq_text and tail_text.endswith(stop_seq_text):
+                return True
+        return False
+
+logging.getLogger("uvicorn").handlers.clear()
+logging.getLogger("uvicorn.error").handlers.clear()
+logging.getLogger("uvicorn.access").handlers.clear()
+logging.getLogger("uvicorn").propagate = False
+logging.getLogger("uvicorn.error").propagate = False
+logging.getLogger("uvicorn.access").propagate = False
+logging.getLogger("uvicorn").setLevel(logging.CRITICAL)
+logging.getLogger("uvicorn.error").setLevel(logging.CRITICAL)
+logging.getLogger("uvicorn.access").setLevel(logging.CRITICAL)
+logging.getLogger("fastapi").setLevel(logging.CRITICAL)
+logging.getLogger("transformers").setLevel(logging.CRITICAL)
+logging.getLogger().handlers.clear()
+logging.getLogger().addHandler(logging.NullHandler())
+
+API_KEY_NAME = "X-API-Key"
+api_key_header = APIKeyHeader(name=API_KEY_NAME)
+API_KEY = os.environ.get("API_KEY", "YOUR_SECRET_API_KEY")
+
+MODEL_NAME = os.environ.get("MODEL_NAME", "jnjj/multi-dataset-model")
+SYSTEM_PROMPT = os.environ.get("SYSTEM_PROMPT", "Eres un asistente profesional y servicial.")
+MAX_CONTEXT_TOKENS = int(os.environ.get("MAX_CONTEXT_TOKENS", 1024))
+MAX_GENERATION_TOKENS = int(os.environ.get("MAX_GENERATION_TOKENS", 512))
+MAX_CONCURRENT_GENERATIONS = int(os.environ.get("MAX_CONCURRENT_GENERATIONS", 4))
+TRUST_REMOTE_CODE = os.environ.get("TRUST_REMOTE_CODE", "false").lower() == "true"
+ENABLE_FLASH_ATTENTION_2 = os.environ.get("ENABLE_FLASH_ATTENTION_2", "false").lower() == "true"
+TORCH_DTYPE_STR = os.environ.get("TORCH_DTYPE", "float16")
+TORCH_DTYPE = getattr(torch, TORCH_DTYPE_STR.lower(), torch.float16)
+if TORCH_DTYPE_STR.lower() == "bfloat16" and (not torch.cuda.is_available() or (torch.cuda.is_available() and not torch.cuda.get_device_properties(0).has_bfloat16)):
+     TORCH_DTYPE = torch.float16
+elif not isinstance(TORCH_DTYPE, torch.dtype):
+     TORCH_DTYPE = torch.float16
+
+global_model = None
+global_tokenizer = None
+global_tokens: Dict[str, Optional[int]] = {}
+executor = ThreadPoolExecutor(max_workers=MAX_CONCURRENT_GENERATIONS)
+generation_semaphore = asyncio.Semaphore(MAX_CONCURRENT_GENERATIONS)
+
+REQUEST_RATE_LIMIT = int(os.environ.get("REQUEST_RATE_LIMIT", 60))
+RATE_LIMIT_WINDOW_SECONDS = int(os.environ.get("RATE_LIMIT_WINDOW_SECONDS", 60))
+request_counts: Dict[str, int] = {}
+
+async def rate_limit_reset_task():
+    while True:
+        await asyncio.sleep(RATE_LIMIT_WINDOW_SECONDS)
+        request_counts.clear()
+
+class GenerateRequest(BaseModel):
+    input_text: str
+    history: Optional[List[Dict[str, str]]] = None
+    stream: bool = True
+    temperature: float = Field(1.0, ge=0.0, le=2.0)
+    top_k: int = Field(50, ge=0)
+    top_p: float = Field(1.0, ge=0.0, le=1.0)
+    repetition_penalty: float = Field(1.0, ge=0.0)
+    frequency_penalty: float = Field(0.0, ge=0.0)
+    presence_penalty: float = Field(0.0, ge=0.0)
+    num_beams: int = Field(1, ge=1)
+    length_penalty: float = Field(1.0, ge=0.0)
+    no_repeat_ngram_size: int = Field(0, ge=0)
+    early_stopping: bool = False
+    do_sample: bool = True
+    use_mirostat: bool = False
+    mirostat_tau: float = Field(5.0, ge=0.0)
+    mirostat_eta: float = Field(0.1, ge=0.0)
+    max_new_tokens: int = Field(MAX_GENERATION_TOKENS, ge=1)
+    system_prompt: Optional[str] = None
+    seed: Optional[int] = None
+    stop_sequences: Optional[List[str]] = None
+    tokenize_only: bool = False
+    strip_trailing_whitespace: bool = False
+    remove_incomplete_sentences: bool = False
+    num_return_sequences: int = Field(1, ge=1, le=5)
+    bad_words_ids: Optional[List[List[int]]] = None
+    forced_bos_token_id: Optional[int] = None
+    forced_eos_token_id: Optional[int] = None
+    renormalize_logits: Optional[bool] = None
+    suppress_tokens: Optional[List[int]] = None
+    begin_suppress_tokens: Optional[List[int]] = None
+    end_suppress_tokens: Optional[List[int]] = None
+    encoder_no_repeat_ngram_size: int = Field(0, ge=0)
+    min_length: int = Field(0, ge=0)
+    max_length: Optional[int] = None
+    exponential_decay_length_penalty: Optional[Tuple[float, int, float]] = None
+    use_cache: bool = True
+    top_k_dynamic: Optional[int] = None
+    top_p_dynamic: Optional[float] = None
+    typical_p: float = Field(1.0, ge=0.0, le=1.0)
+    epsilon_cutoff: float = Field(0.0, ge=0.0)
+    eta_cutoff: float = Field(0.0, ge=0.0)
+    temperature_cutoff: Optional[float] = Field(None, ge=0.0)
+    encoder_repetition_penalty: float = Field(1.0, ge=0.0)
+    max_time: Optional[float] = Field(None, ge=0.0)
+    output_watermark: bool = False
+    remove_input_from_output: bool = False
+    eos_token_id_override: Optional[int] = None
+    pad_token_id_override: Optional[int] = None
+    bos_token_id_override: Optional[int] = None
+    repetition_penalty_range: Optional[int] = Field(None, ge=0)
+    diversity_penalty: float = Field(0.0, ge=0.0)
+    num_beam_groups: int = Field(1, ge=1)
+    return_dict_in_generate: bool = False
+    output_attentions: bool = False
+    output_hidden_states: bool = False
+    output_scores: bool = False
+    return_token_logprobs: bool = False
+    return_text_from_sequence: bool = True
+    length_normalization_factor: Optional[float] = None
+    min_new_tokens: int = Field(0, ge=0)
+    do_normalize_logits: bool = False
+    return_generation_inputs: bool = False
+    return_unused_generate_parameters: bool = False
+    use_fast_tokenizer: Optional[bool] = None
+    model_kwargs: Optional[Dict[str, Any]] = None
+    tokenizer_kwargs: Optional[Dict[str, Any]] = None
+    generation_kwargs: Optional[Dict[str, Any]] = None
+    prompt_lookup_num_tokens: Optional[int] = Field(None, ge=0)
+    tfs_lambda: Optional[float] = Field(None, ge=0.0, le=1.0)
+    top_p_min: Optional[float] = Field(None, ge=0.0, le=1.0)
+    typical_p_min: Optional[float] = Field(None, ge=0.0, le=1.0)
+    sequence_bias: Optional[Dict[int, float]] = None
+    negative_prompt_ids: Optional[List[int]] = None
+    return_only_text: bool = False
+
+    @validator('stop_sequences')
+    def validate_stop_sequences(cls, v):
+        if v is not None:
+            if not all(isinstance(seq, str) for seq in v):
+                raise ValueError('Each stop sequence must be a string')
+        return v
+
+    @validator('bad_words_ids')
+    def validate_bad_words_ids(cls, v):
+        if v is not None:
+            if not all(isinstance(word_id_list, list) and all(isinstance(token_id, int) for token_id in word_id_list) for word_id_list in v):
+                raise ValueError('bad_words_ids must be a list of lists of integers')
+        return v
+
+    @validator('exponential_decay_length_penalty')
+    def validate_exponential_decay_length_penalty(cls, v):
+        if v is not None:
+            if not (isinstance(v, (list, tuple)) and len(v) == 3 and
+                    isinstance(v[0], (int, float)) and v[0] > 0 and
+                    isinstance(v[1], int) and v[1] >= 0 and
+                    isinstance(v[2], (int, float))):
+                 raise ValueError('exponential_decay_length_penalty must be a tuple/list of 3 numbers (decay_factor, start_index, threshold)')
+        return v
+
+    @validator('sequence_bias')
+    def validate_sequence_bias(cls, v):
+         if v is not None:
+              if not all(isinstance(k, int) and isinstance(val, float) for k, val in v.items()):
+                   raise ValueError('sequence_bias must be a dictionary with integer keys and float values')
+         return v
+
+    @validator('negative_prompt_ids')
+    def validate_negative_prompt_ids(cls, v):
+         if v is not None:
+              if not all(isinstance(token_id, int) for token_id in v):
+                   raise ValueError('negative_prompt_ids must be a list of integers')
+         return v
+
+class TokenizeRequest(BaseModel):
+    text: Union[str, List[str]]
+    add_special_tokens: bool = True
+    is_split_into_words: bool = False
+    return_token_type_ids: bool = False
+    padding: Union[bool, str] = False
+    truncation: Union[bool, str] = False
+    max_length: Optional[int] = None
+    return_tensors: Optional[str] = None
+    return_attention_mask: Optional[bool] = None
+    return_offsets_mapping: Optional[bool] = None
+    return_length: Optional[bool] = None
+    verbose: bool = False
+    tokenizer_kwargs: Optional[Dict[str, Any]] = None
+    use_fast_tokenizer: Optional[bool] = None
+    chat_format: bool = False
+    return_word_ids: bool = False
+
+class DecodeRequest(BaseModel):
+    token_ids: List[int]
+    skip_special_tokens: bool = True
+    clean_up_tokenization_spaces: bool = True
+    decode_kwargs: Optional[Dict[str, Any]] = None
+
+class SystemPromptUpdateRequest(BaseModel):
+    system_prompt: str
+
+class ModelReloadRequest(BaseModel):
+    model_name: Optional[str] = None
+    trust_remote_code: Optional[bool] = None
+    enable_flash_attention_2: Optional[bool] = None
+    torch_dtype: Optional[str] = None
+    model_kwargs: Optional[Dict[str, Any]] = None
+    tokenizer_kwargs: Optional[Dict[str, Any]] = None
+
+def authenticate_api_key(api_key: str = Depends(api_key_header)):
+    if api_key != API_KEY:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API Key")
+    return api_key
+
+def format_conversation(input_text: str, history: Optional[List[Dict[str, str]]], system_prompt: Optional[str], tokenizer: AutoTokenizer) -> str:
+    full_history: List[Dict[str, str]] = []
+    used_system_prompt = system_prompt if system_prompt is not None else SYSTEM_PROMPT
+    if used_system_prompt:
+        full_history.append({"role": "system", "content": used_system_prompt})
+    if history:
+        full_history.extend(history)
+    full_history.append({"role": "user", "content": input_text})
+    if tokenizer and hasattr(tokenizer, 'apply_chat_template') and tokenizer.chat_template:
+         try:
+              return tokenizer.apply_chat_template(full_history, tokenize=False, add_generation_prompt=True)
+         except Exception:
+              pass
+    formatted_text = ""
+    for message in full_history:
+        if message["role"] == "system":
+            formatted_text += f"{message['content'].strip()}\n\n"
+        elif message["role"] == "user":
+            formatted_text += f"Usuario: {message['content'].strip()}\n"
+        elif message["role"] == "assistant":
+             formatted_text += f"Bot: {message['content'].strip()}\n"
+    if not formatted_text.strip().endswith("Bot:"):
+         formatted_text = formatted_text.strip() + "\nBot:"
+    return formatted_text.strip()
+
+def truncate_encoded_ids(input_ids: torch.Tensor, max_length: int) -> torch.Tensor:
+    if input_ids.shape[-1] > max_length:
+        return input_ids[:, -max_length:]
+    return input_ids
+
+def apply_seed(seed: Optional[int]):
+    if seed is not None:
+        torch.manual_seed(seed)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(seed)
+        random.seed(seed)
+
+def get_generation_params(req: GenerateRequest, initial_ids: torch.Tensor, tokenizer: AutoTokenizer, device: str) -> Tuple[GenerationConfig, StoppingCriteriaList, LogitsProcessorList, Dict[str, Any]]:
+    initial_len = initial_ids.shape[-1]
+    logits_processor = LogitsProcessorList()
+    if req.min_length is not None and req.min_length > 0:
+        eos_token_id = req.eos_token_id_override if req.eos_token_id_override is not None else global_tokens.get("eos_token_id", -1)
+        logits_processor.append(MinLengthLogitsProcessor(initial_len + req.min_length, eos_token_id))
+
+    stopping_criteria = StoppingCriteriaList()
+    max_gen_len = req.max_new_tokens if req.max_new_tokens is not None else MAX_GENERATION_TOKENS
+    effective_max_length = req.max_length if req.max_length is not None and req.max_length > 0 else initial_len + max_gen_len
+    if effective_max_length > 0:
+        stopping_criteria.append(MaxLengthCriteria(effective_max_length))
+
+    if req.stop_sequences:
+         stop_sequence_criteria_instance = StopSequenceCriteria(req.stop_sequences, tokenizer)
+         if stop_sequence_criteria_instance.stop_sequence_ids or stop_sequence_criteria_instance.stop_sequences_text:
+             stopping_criteria.append(stop_sequence_criteria_instance)
+
+    gen_cfg_params = {
+        "temperature": req.temperature,
+        "top_k": req.top_k,
+        "top_p": req.top_p,
+        "repetition_penalty": req.repetition_penalty,
+        "num_beams": req.num_beams if not req.stream else 1,
+        "length_penalty": req.length_penalty,
+        "no_repeat_ngram_size": req.no_repeat_ngram_size,
+        "early_stopping": req.early_stopping,
+        "do_sample": req.do_sample,
+        "max_new_tokens": req.max_new_tokens,
+        "eos_token_id": req.eos_token_id_override if req.eos_token_id_override is not None else global_tokens.get("eos_token_id"),
+        "pad_token_id": req.pad_token_id_override if req.pad_token_id_override is not None else global_tokens.get("pad_token_id"),
+        "bos_token_id": req.bos_token_id_override if req.bos_token_id_override is not None else (global_tokenizer.bos_token_id if global_tokenizer else None),
+        "num_return_sequences": req.num_return_sequences if not req.stream else 1,
+        "forced_bos_token_id": req.forced_bos_token_id,
+        "forced_eos_token_id": req.forced_eos_token_id,
+        "encoder_no_repeat_ngram_size": req.encoder_no_repeat_ngram_size,
+        "min_length": req.min_length,
+        "max_length": effective_max_length,
+        "exponential_decay_length_penalty": req.exponential_decay_length_penalty,
+        "use_cache": req.use_cache,
+        "top_k_dynamic": req.top_k_dynamic,
+        "top_p_dynamic": req.top_p_dynamic,
+        "typical_p": req.typical_p,
+        "epsilon_cutoff": req.epsilon_cutoff,
+        "eta_cutoff": req.eta_cutoff,
+        "temperature_cutoff": req.temperature_cutoff,
+        "encoder_repetition_penalty": req.encoder_repetition_penalty,
+        "max_time": req.max_time,
+        "diversity_penalty": req.diversity_penalty,
+        "num_beam_groups": req.num_beam_groups if not req.stream else 1,
+        "length_normalization_factor": req.length_normalization_factor,
+        "min_new_tokens": req.min_new_tokens,
+    }
+
+    direct_generate_params = {
+        "bad_words_ids": req.bad_words_ids,
+        "sequence_bias": req.sequence_bias,
+        "negative_prompt_ids": req.negative_prompt_ids,
+        "return_dict_in_generate": req.return_dict_in_generate,
+        "output_scores": req.output_scores and not req.stream,
+        "output_attentions": req.output_attentions and not req.stream,
+        "output_hidden_states": req.output_hidden_states and not req.stream,
+        "return_generation_inputs": req.return_generation_inputs,
+        "return_unused_generate_parameters": req.return_unused_generate_parameters,
+        "frequency_penalty": req.frequency_penalty,
+        "presence_penalty": req.presence_penalty,
+        "output_watermark": req.output_watermark,
+        "do_normalize_logits": req.do_normalize_logits,
+        "prompt_lookup_num_tokens": req.prompt_lookup_num_tokens,
+        "tfs_lambda": req.tfs_lambda,
+        "top_p_min": req.top_p_min,
+        "typical_p_min": req.typical_p_min,
+    }
+    if req.use_mirostat:
+        direct_generate_params['use_mirostat_mode'] = 1
+        direct_generate_params['mirostat_tau'] = req.mirostat_tau
+        direct_generate_params['mirostat_eta'] = req.mirostat_eta
+    else:
+         direct_generate_params['use_mirostat_mode'] = 0
+
+    if req.model_kwargs:
+         direct_generate_params['model_kwargs'] = req.model_kwargs
+
+    if req.generation_kwargs:
+        direct_generate_params.update(req.generation_kwargs)
+
+    gen_cfg_params_filtered = {k: v for k, v in gen_cfg_params.items() if v is not None}
+    direct_generate_params_filtered = {k: v for k, v in direct_generate_params.items() if v is not None}
+
+    try:
+        gen_cfg = GenerationConfig(**gen_cfg_params_filtered)
+    except Exception as e:
+        raise ValueError(f"Error creating GenerationConfig: {e}. Parameters: {gen_cfg_params_filtered}")
+
+    return gen_cfg, stopping_criteria, logits_processor, direct_generate_params_filtered
+
+def post_process_text(text: str, strip_trailing_whitespace: bool, remove_incomplete_sentences: bool) -> str:
+    if strip_trailing_whitespace:
+        text = text.rstrip()
+    if remove_incomplete_sentences:
+        for terminator in ['.', '!', '?', '\n']:
+            last_terminator = text.rfind(terminator)
+            if last_terminator != -1:
+                text = text[:last_terminator + 1]
+                break
+    return text
+
+async def stream_generation_logic(req: GenerateRequest, initial_ids: torch.Tensor, gen_cfg: GenerationConfig, stopping_criteria: StoppingCriteriaList, logits_processor: LogitsProcessorList, direct_generate_params: Dict[str, Any], device: str) -> AsyncGenerator[str, None]:
+    queue = SyncToAsyncQueue()
+    streamer = QueueTextStreamer(queue, global_tokenizer, skip_prompt=True)
+    initial_len = initial_ids.shape[-1]
+
+    generation_future = asyncio.get_event_loop().run_in_executor(
+        executor,
+        lambda: global_model.generate(
+            initial_ids,
+            generation_config=gen_cfg,
+            stopping_criteria=stopping_criteria,
+            logits_processor=logits_processor,
+            streamer=streamer,
+            return_dict_in_generate=True,
+            **direct_generate_params,
+        )
+    )
+
+    try:
+        accumulated_text = ""
+        while True:
+            try:
+                 item = await queue.get()
+                 if item["type"] == "stream_end_signal":
+                     break
+                 if item["type"] == "token":
+                     yield json.dumps(item) + "\n"
+                     accumulated_text += item.get("text", "")
+                 elif item["type"] == "error":
+                     yield json.dumps(item) + "\n"
+            except asyncio.QueueClosed:
+                 break
+
+        output = await asyncio.wrap_future(generation_future)
+
+    except Exception as e:
+         yield json.dumps({"type": "error", "message": f"Stream processing or generation error: {e}"}) + "\n"
+         if generation_future and not generation_future.done():
+              generation_future.cancel()
+         output = None
+    finally:
+         queue.close()
+
+    final_ids_generated = []
+    full_sequence_ids = initial_ids.tolist()[0]
+    total_generated_tokens = 0
+    finish_reason = "error"
+    final_text_processed = ""
+
+    if output is not None and isinstance(output, GenerationOutput) and output.sequences is not None and output.sequences.shape[0] > 0:
+        final_sequence = output.sequences[0]
+        final_ids_generated = final_sequence[initial_len:].tolist()
+        total_generated_tokens = len(final_ids_generated)
+        full_sequence_ids = final_sequence.tolist()
+
+        final_text_raw = global_tokenizer.decode(final_ids_generated, skip_special_tokens=True)
+        if req.stop_sequences:
+             for stop_seq in req.stop_sequences:
+                  if stop_seq and stop_seq in final_text_raw:
+                       final_text_raw = final_text_raw.split(stop_seq, 1)[0]
+                       break
+        final_text_processed = post_process_text(final_text_raw, req.strip_trailing_whitespace, req.remove_incomplete_sentences)
+
+        if hasattr(output, 'stopping_criteria') and output.stopping_criteria is not None:
+             stopped_by = output.stopping_criteria
+             if any(isinstance(c, MaxLengthCriteria) for c in stopped_by):
+                  finish_reason = "max_length"
+             elif any(isinstance(c, StopSequenceCriteria) for c in stopped_by):
+                  finish_reason = "stop_sequence"
+             else:
+                   finish_reason = "stop"
+        if finish_reason == "error":
+             eos_token_id = req.eos_token_id_override if req.eos_token_id_override is not None else global_tokens.get("eos_token_id")
+             if eos_token_id is not None and final_ids_generated and final_ids_generated[-1] == eos_token_id:
+                 finish_reason = "eos_token"
+             elif total_generated_tokens >= (req.max_new_tokens if req.max_new_tokens is not None else MAX_GENERATION_TOKENS):
+                  finish_reason = "max_new_tokens"
+             elif req.max_length is not None and len(full_sequence_ids) >= req.max_length:
+                  finish_reason = "max_length"
+             elif req.stop_sequences and any(seq in post_process_text(global_tokenizer.decode(final_ids_generated, skip_special_tokens=True), False, False) for seq in req.stop_sequences):
+                  finish_reason = "stop_sequence"
+             elif hasattr(output, 'max_time_exceeded') and output.max_time_exceeded:
+                  finish_reason = "time"
+             else:
+                  finish_reason = "unknown"
+
+    else:
+         finish_reason = "error_or_no_output"
+
+    final_payload: Dict[str, Any] = {
+        "type": "done",
+        "total_prompt_tokens": initial_len,
+        "total_generated_tokens": total_generated_tokens,
+        "total_sequence_tokens": len(full_sequence_ids),
+        "final_text": final_text_processed,
+        "finish_reason": finish_reason
+    }
+    if not req.remove_input_from_output:
+         final_payload["full_sequence_token_ids"] = full_sequence_ids
+         final_payload["generated_token_ids"] = final_ids_generated
+    yield json.dumps(final_payload) + "\n"
+
+    await cleanup(device)
+
+async def non_stream_generation_logic(req: GenerateRequest, initial_ids: torch.Tensor, gen_cfg: GenerationConfig, stopping_criteria: StoppingCriteriaList, logits_processor: LogitsProcessorList, direct_generate_params: Dict[str, Any], device: str) -> Dict[str, Any]:
+    try:
+        generate_params = gen_cfg.to_dict()
+        generate_params.update({
+             "input_ids": initial_ids,
+             "stopping_criteria": stopping_criteria if stopping_criteria else None,
+             "logits_processor": logits_processor if logits_processor else None,
+             **direct_generate_params,
+        })
+
+        if req.generation_kwargs:
+            generate_params.update(req.generation_kwargs)
+
+        with torch.no_grad():
+             out = await asyncio.to_thread(global_model.generate, **generate_params)
+
+        generated_data = []
+        initial_len = initial_ids.shape[-1]
+        eos_token_id = req.eos_token_id_override if req.eos_token_id_override is not None else global_tokens.get("eos_token_id")
+        effective_max_length = req.max_length if req.max_length is not None and req.max_length > 0 else initial_len + (req.max_new_tokens if req.max_new_tokens is not None else MAX_GENERATION_TOKENS)
+
+        for i in range(req.num_return_sequences):
+            if i >= out.sequences.shape[0]:
+                 break
+            sequence = out.sequences[i]
+            generated_ids_tensor = sequence[initial_len:]
+            full_sequence_ids = sequence.tolist()
+            text_raw = global_tokenizer.decode(generated_ids_tensor, skip_special_tokens=True)
+
+            if req.stop_sequences:
+                 for stop_seq in req.stop_sequences:
+                      if stop_seq and stop_seq in text_raw:
+                           text_raw = text_raw.split(stop_seq, 1)[0]
+                           break
+
+            text_processed = post_process_text(text_raw, req.strip_trailing_whitespace, req.remove_incomplete_sentences)
+            generated_tokens_count = len(generated_ids_tensor)
+
+            finish_reason = "unknown"
+            if hasattr(out, 'stopping_criteria') and out.stopping_criteria is not None:
+                 stopped_by_criteria = out.stopping_criteria
+                 if any(isinstance(c, MaxLengthCriteria) for c in stopped_by_criteria):
+                     finish_reason = "max_length"
+                 elif any(isinstance(c, StopSequenceCriteria) for c in stopped_by_criteria):
+                      finish_reason = "stop_sequence"
+                 else:
+                       finish_reason = "stop"
+            if finish_reason == "unknown":
+                 if eos_token_id is not None and generated_ids_tensor.numel() > 0 and generated_ids_tensor[-1] == eos_token_id:
+                      finish_reason = "eos_token"
+                 elif len(full_sequence_ids) >= effective_max_length:
+                      if req.max_new_tokens is not None and generated_tokens_count >= req.max_new_tokens:
+                           finish_reason = "max_new_tokens"
+                      elif req.max_length is not None and len(full_sequence_ids) >= req.max_length:
+                           finish_reason = "max_length"
+                      else:
+                           finish_reason = "length"
+                 elif req.stop_sequences and any(seq in text_processed for seq in req.stop_sequences):
+                       finish_reason = "stop_sequence"
+                 elif hasattr(out, 'max_time_exceeded') and out.max_time_exceeded:
+                       finish_reason = "time"
+                 else:
+                       finish_reason = "stop"
+
+            item_data: Dict[str, Any] = {
+                "text": text_processed if req.return_text_from_sequence else None,
+                "token_ids": generated_ids_tensor.tolist(),
+                "generated_tokens_count": generated_tokens_count,
+                "finish_reason": finish_reason
+            }
+            if not req.remove_input_from_output:
+                 item_data["full_sequence_token_ids"] = full_sequence_ids
+
+            if req.output_scores and hasattr(out, 'scores') and out.scores is not None:
+                 item_data["scores"] = [score.tolist() for score in out.scores] if isinstance(out.scores, tuple) else out.scores.tolist()
+                 if req.return_token_logprobs:
+                      try:
+                          log_probs_list = []
+                          if hasattr(out, 'scores') and out.scores is not None:
+                                raw_scores = out.scores
+                                if isinstance(raw_scores, tuple):
+                                    if raw_scores and raw_scores[0].ndim == 2:
+                                        step_logits_list = [s[i] for s in raw_scores]
+                                    elif raw_scores:
+                                         step_logits_list = raw_scores
+                                    else:
+                                         step_logits_list = []
+                                elif raw_scores.ndim == 3:
+                                     step_logits_list = [raw_scores[step_idx, i] for step_idx in range(raw_scores.shape[0])]
+                                elif raw_scores.ndim == 2:
+                                     step_logits_list = [raw_scores[step_idx] for step_idx in range(raw_scores.shape[0])]
+                                else:
+                                     step_logits_list = []
+
+                                step_log_probs = [F.log_softmax(l, dim=-1) for l in step_logits_list if l is not None and l.numel() > 0]
+                                for step_idx, token_id in enumerate(generated_ids_tensor.tolist()):
+                                   if step_idx < len(step_log_probs) and token_id < step_log_probs[step_idx].shape[-1]:
+                                        log_probs_list.append(float(step_log_probs[step_idx][token_id].item()))
+                                   else:
+                                        log_probs_list.append(None)
+                                item_data["token_logprobs"] = log_probs_list
+                          else:
+                               item_data["token_logprobs"] = "Scores not available."
+                      except Exception:
+                           item_data["token_logprobs"] = "Error calculating logprobs."
+
+            if req.output_attentions and hasattr(out, 'attentions') and out.attentions is not None:
+                 item_data["attentions"] = "Attentions data is large and complex."
+            if req.output_hidden_states and hasattr(out, 'hidden_states') and out.hidden_states is not None:
+                 item_data["hidden_states"] = "Hidden states data is large and complex."
+            if req.output_watermark and hasattr(out, 'watermark') and out.watermark is not None:
+                 item_data["watermark"] = out.watermark[i] if isinstance(out.watermark, list) and len(out.watermark) > i else out.watermark
+
+            generated_data.append(item_data)
+
+        response_payload: Dict[str, Any] = {
+             "prompt_tokens": initial_len,
+             "generated_sequences": generated_data,
+        }
+        if req.num_return_sequences == 1 and generated_data:
+             response_payload["total_tokens"] = response_payload["prompt_tokens"] + generated_data[0]["generated_tokens_count"]
+
+        if req.return_dict_in_generate and isinstance(out, GenerationOutput):
+             raw_out_dict = {}
+             for key in out.keys():
+                  if key not in ['sequences', 'scores', 'attentions', 'hidden_states', 'past_key_values', 'watermark', 'sequences_scores', 'beam_indices', 'stopping_criteria']:
+                       value = out[key]
+                       if isinstance(value, torch.Tensor):
+                            raw_out_dict[key] = value.tolist()
+                       else:
+                            raw_out_dict[key] = value
+             response_payload["raw_generate_output"] = raw_out_dict
+
+        await cleanup(device)
+        return response_payload
+
+    except Exception as e:
+         await cleanup(device)
+         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Generation error: {e}")
+
+async def cleanup(device: str):
+    if device == "cuda":
+        torch.cuda.empty_cache()
+    gc.collect()
+
+app = FastAPI(title="Chatbot Profesional API", version="1.0.0")
+
+@app.on_event("startup")
+async def startup_event():
+    asyncio.create_task(load_model())
+    asyncio.create_task(rate_limit_reset_task())
+
+async def load_model():
+    global global_model, global_tokenizer, global_tokens, MODEL_NAME, TRUST_REMOTE_CODE, ENABLE_FLASH_ATTENTION_2, TORCH_DTYPE, TORCH_DTYPE_STR
+    try:
+         TORCH_DTYPE = getattr(torch, TORCH_DTYPE_STR.lower(), torch.float16)
+         if TORCH_DTYPE_STR.lower() == "bfloat16" and (not torch.cuda.is_available() or (torch.cuda.is_available() and not torch.cuda.get_device_properties(0).has_bfloat16)):
+              TORCH_DTYPE = torch.float16
+         elif not isinstance(TORCH_DTYPE, torch.dtype):
+              TORCH_DTYPE = torch.float16
+    except AttributeError:
+         TORCH_DTYPE = torch.float16
+    try:
+        config = AutoConfig.from_pretrained(MODEL_NAME, trust_remote_code=TRUST_REMOTE_CODE)
+        tokenizer_kwargs = {"config": config, "trust_remote_code": TRUST_REMOTE_CODE}
+        global_tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, **tokenizer_kwargs)
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        model_kwargs = {"config": config, "torch_dtype": TORCH_DTYPE, "trust_remote_code": TRUST_REMOTE_CODE}
+        if ENABLE_FLASH_ATTENTION_2 and device == "cuda":
+            try:
+                 if hasattr(config, '_supports_flash_attn_2') and config._supports_flash_attn_2:
+                     model_kwargs['use_flash_attention_2'] = True
+                 elif hasattr(AutoModelForCausalLM, '_supports_flash_attn_2') and AutoModelForCausalLM._supports_flash_attn_2(config):
+                      model_kwargs['use_flash_attention_2'] = True
+            except Exception:
+                 pass
+        global_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, **model_kwargs)
+        global_model.to(device)
+        global_model.eval()
+        global_tokens["eos_token_id"] = global_tokenizer.eos_token_id
+        global_tokens["pad_token_id"] = global_tokenizer.pad_token_id
+        if global_tokens["pad_token_id"] is None and global_tokens["eos_token_id"] is not None:
+             global_tokens["pad_token_id"] = global_tokens["eos_token_id"]
+             if global_model.config.pad_token_id is None:
+                  global_model.config.pad_token_id = global_tokens["pad_token_id"]
+    except Exception:
+        global_model = None
+        global_tokenizer = None
+        global_tokens = {}
+
+html_code = """
+<!DOCTYPE html>
+<html lang="es">
+<head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Chatbot Profesional</title>
+    <style>
+        body { font-family: Arial, sans-serif; margin: 20px; display: flex; flex-direction: column; height: calc(100vh - 40px);}
+        #chatbox { flex-grow: 1; border: 1px solid #ccc; padding: 10px; overflow-y: auto; margin-bottom: 10px; display: flex; flex-direction: column;}
+        .message { margin-bottom: 10px; padding: 8px; border-radius: 5px; max-width: 80%; word-wrap: break-word;}
+        .user-message { background-color: #e1ffc7; align-self: flex-end; }
+        .bot-message { background-color: #f1f0f0; align-self: flex-start; }
+        .system-message { background-color: #e0f7fa; align-self: flex-start; font-size: 0.9em; color: #555; }
+        .error-message { background-color: #ffebee; align-self: flex-start; font-size: 0.9em; color: #d32f2f; }
+        .message strong { display: block; margin-bottom: 4px; }
+        #input-area { display: flex; }
+        #user-input { flex-grow: 1; padding: 8px; box-sizing: border-box; margin-right: 5px;}
+        #send-btn { padding: 8px 15px;}
+    </style>
+</head>
+<body>
+    <h1>Chatbot Profesional (POST API)</h1>
+    <div id="chatbox"></div>
+    <div id="input-area">
+        <input type="text" id="user-input" placeholder="Escribe tu mensaje aquí..." autocomplete="off"/>
+        <button id="send-btn">Enviar</button>
+    </div>
+    <script>
+        const chatbox = document.getElementById('chatbox');
+        const userInput = document.getElementById('user-input');
+        const sendBtn = document.getElementById('send-btn');
+        let conversationHistory = [];
+        const DEFAULT_SYSTEM_PROMPT = "Eres un asistente profesional y servicial.";
+        let currentSystemPrompt = DEFAULT_SYSTEM_PROMPT;
+        let currentBotMessageElement = null;
+
+        function appendMessage(sender, text, type = 'bot', isStreaming = false) {
+            let msgDiv;
+            let contentElement;
+            if (isStreaming && currentBotMessageElement && type === 'bot') {
+                 contentElement = currentBotMessageElement.querySelector('.message-content');
+                 if (contentElement) {
+                     contentElement.textContent += text;
+                 } else {
+                     msgDiv = document.createElement('div');
+                     msgDiv.classList.add('message', `${type}-message`);
+                     msgDiv.innerHTML = `<strong>${sender}:</strong> <span class="message-content">${text}</span>`;
+                     chatbox.appendChild(msgDiv);
+                     currentBotMessageElement = msgDiv;
+                 }
+            } else {
+                msgDiv = document.createElement('div');
+                msgDiv.classList.add('message', `${type}-message`);
+                msgDiv.innerHTML = `<strong>${sender}:</strong> <span class="message-content">${text}</span>`;
+                chatbox.appendChild(msgDiv);
+                if (sender === 'Bot' && isStreaming) {
+                    currentBotMessageElement = msgDiv;
+                } else {
+                    currentBotMessageElement = null;
+                }
+            }
+            chatbox.scrollTop = chatbox.scrollHeight;
+        }
+
+        function updateHistory(role, content) {
+             conversationHistory.push({ "role": role, "content": content });
+             const maxHistorySize = 20;
+             if (conversationHistory.length > maxHistorySize) {
+                  const systemPrompt = conversationHistory.find(m => m.role === 'system');
+                  let startIndex = Math.max(1, conversationHistory.length - maxHistorySize);
+                  let newHistory = conversationHistory.slice(startIndex);
+                  if (systemPrompt && newHistory[0]?.role !== 'system') {
+                       newHistory.unshift(systemPrompt);
+                  }
+                  conversationHistory = newHistory;
+             }
+        }
+
+        async function sendMessage() {
+            const text = userInput.value.trim();
+            if (!text) {
+                return;
+            }
+            appendMessage('Usuario', text, 'user');
+            updateHistory("user", text);
+            userInput.value = '';
+            sendBtn.disabled = true;
+            currentBotMessageElement = null;
+
+            const isStreamingRequest = true;
+            const requestPayload = {
+                input_text: text,
+                history: conversationHistory,
+                system_prompt: currentSystemPrompt,
+                stream: isStreamingRequest,
+                temperature: 0.7,
+                top_k: 0,
+                top_p: 0.9,
+                max_new_tokens: 512,
+            };
+
+            try {
+                const response = await fetch('/generate', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-API-Key': 'YOUR_SECRET_API_KEY'
+                    },
+                    body: JSON.stringify(requestPayload),
+                });
+
+                if (!response.ok) {
+                    const errorData = await response.json();
+                    throw new Error(`API Error: ${response.status} ${response.statusText} - ${errorData.detail || errorData.error || 'Unknown error'}`);
+                }
+
+                if (isStreamingRequest) {
+                    const reader = response.body.getReader();
+                    const decoder = new TextDecoder();
+                    let buffer = '';
+                    let finalBotText = "";
+                    appendMessage('Bot', '', 'bot', true);
+
+                    while (true) {
+                        const { value, done } = await reader.read();
+                        if (done) break;
+                        buffer += decoder.decode(value, { stream: true });
+                        const lines = buffer.split('\n');
+                        buffer = lines.pop();
+                        for (const line of lines) {
+                            if (line.trim() === '') continue;
+                            try {
+                                const data = JSON.parse(line);
+                                if (data.type === 'token') {
+                                    appendMessage('Bot', data.text, 'bot', true);
+                                    finalBotText += data.text;
+                                } else if (data.type === 'done') {
+                                     appendMessage('System', `Generated ${data.total_generated_tokens} tokens. Finish reason: ${data.finish_reason}`, 'system');
+                                    updateHistory("assistant", data.final_text !== undefined ? data.final_text : finalBotText);
+                                } else if (data.type === 'error') {
+                                    appendMessage('Error', `Stream Error: ${data.message}`, 'error');
+                                    updateHistory("assistant", finalBotText || "Error during streaming.");
+                                    finalBotText = "";
+                                    reader.cancel();
+                                    return;
+                                }
+                            } catch (e) {
+                                console.error('Failed to parse stream chunk:', e, line);
+                                appendMessage('Error', 'Failed to process stream chunk.', 'error');
+                                updateHistory("assistant", finalBotText || "Error processing stream.");
+                                finalBotText = "";
+                                reader.cancel();
+                                return;
+                            }
+                        }
+                    }
+                    if (buffer.trim() !== '') {
+                         try {
+                              const data = JSON.parse(buffer);
+                               if (data.type === 'token') {
+                                   appendMessage('Bot', data.text, 'bot', true);
+                                   finalBotText += data.text;
+                               } else if (data.type === 'done') {
+                                    appendMessage('System', `Generated ${data.total_generated_tokens} tokens. Finish reason: ${data.finish_reason}`, 'system');
+                                    updateHistory("assistant", data.final_text !== undefined ? data.final_text : finalBotText);
+                               } else if (data.type === 'error') {
+                                   appendMessage('Error', `Stream Error: ${data.message}`, 'error');
+                                   updateHistory("assistant", finalBotText || "Error during streaming.");
+                                   finalBotText = "";
+                               }
+                         } catch (e) {
+                             console.error('Failed to parse remaining buffer:', e, buffer);
+                             appendMessage('Error', 'Failed to process remaining stream data.', 'error');
+                              updateHistory("assistant", finalBotText || "Error processing remaining stream data.");
+                              finalBotText = "";
+                         }
+                    }
+                } else {
+                    const data = await response.json();
+                    if (data.text !== undefined && Object.keys(data).length === 1) {
+                         appendMessage('Bot', data.text, 'bot');
+                         updateHistory("assistant", data.text);
+                    } else if (data.generated_sequences && data.generated_sequences.length > 0) {
+                         const generatedText = data.generated_sequences[0].text;
+                         appendMessage('Bot', generatedText, 'bot');
+                         updateHistory("assistant", generatedText);
+                          appendMessage('System', `Generated ${data.generated_sequences[0].generated_tokens_count} tokens. Finish reason: ${data.generated_sequences[0].finish_reason}`, 'system');
+                    } else if (data.error) {
+                        appendMessage('Error', `API Error: ${data.error}`, 'error');
+                    } else {
+                        appendMessage('Error', 'Received unexpected non-streaming response format.', 'error');
+                        console.error("Unexpected non-streaming response:", data);
+                    }
+                }
+            } catch (error) {
+                console.error('Send message error:', error);
+                appendMessage('Error', error.message || 'An unknown error occurred.', 'error');
+                 updateHistory("assistant", "Error generating response.");
+            } finally {
+                 sendBtn.disabled = false;
+                 currentBotMessageElement = null;
+            }
+        }
+        sendBtn.onclick = sendMessage;
+        userInput.addEventListener('keypress', function(event) {
+            if (event.key === 'Enter' && !sendBtn.disabled) {
+                event.preventDefault();
+                sendMessage();
+            }
+        });
+        document.addEventListener('DOMContentLoaded', () => {
+             appendMessage('System', 'Chatbot cargado. ¡Hola!', 'system');
+        });
+    </script>
+</body>
+</html>
+"""
+
+@app.get("/", response_class=HTMLResponse, summary="Interactive HTML interface")
+async def root():
+    return HTMLResponse(content=html_code)
+
+@app.get("/health", summary="Check API health and model status")
+async def health():
+    if global_model is None or global_tokenizer is None:
+        status_detail = "loading model"
+        if global_tokenizer is None and global_model is not None:
+             status_detail = "tokenizer not loaded"
+        elif global_model is None and global_tokenizer is not None:
+             status_detail = "model not loaded"
+        elif global_model is None and global_tokenizer is None:
+             status_detail = "model and tokenizer not loaded"
+        return JSONResponse(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, content={"status": status_detail, "model_loaded": global_model is not None, "tokenizer_loaded": global_tokenizer is not None})
+    try:
+        dummy_input = "Hello"
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        encoded_dummy = global_tokenizer(dummy_input, return_tensors="pt").to(device)
+        with torch.no_grad():
+            _ = global_model.generate(encoded_dummy.input_ids[:, -1:], max_new_tokens=1, num_beams=1, do_sample=False)
+        return {"status": "ok", "model_loaded": True, "tokenizer_loaded": True, "generation_test": "successful"}
+    except Exception as e:
+        return JSONResponse(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, content={"status": "model_loaded_but_generation_failed", "model_loaded": True, "tokenizer_loaded": True, "generation_test_error": str(e)})
+
+
+@app.get("/config", summary="Get API configuration", dependencies=[Depends(authenticate_api_key)])
+async def get_config():
+    torch_dtype_str_out = str(TORCH_DTYPE).split('.')[-1] if isinstance(TORCH_DTYPE, torch.dtype) else str(TORCH_DTYPE)
+    return JSONResponse({
+        "model_name": MODEL_NAME,
+        "system_prompt_default": SYSTEM_PROMPT,
+        "max_context_tokens": MAX_CONTEXT_TOKENS,
+        "max_generation_tokens": MAX_GENERATION_TOKENS,
+        "cuda_available": torch.cuda.is_available(),
+        "model_loaded": global_model is not None,
+        "tokenizer_loaded": global_tokenizer is not None,
+        "max_concurrent_generations": MAX_CONCURRENT_GENERATIONS,
+        "trust_remote_code": TRUST_REMOTE_CODE,
+        "enable_flash_attention_2": ENABLE_FLASH_ATTENTION_2,
+        "torch_dtype": torch_dtype_str_out,
+        "eos_token_id": global_tokens.get("eos_token_id"),
+        "pad_token_id": global_tokens.get("pad_token_id"),
+        "bos_token_id": global_tokenizer.bos_token_id if global_tokenizer else None,
+        "request_rate_limit": REQUEST_RATE_LIMIT,
+        "rate_limit_window_seconds": RATE_LIMIT_WINDOW_SECONDS,
+    })
+
+@app.get("/model/info", summary="Get model information", dependencies=[Depends(authenticate_api_key)])
+async def get_model_info():
+    if global_model is None or global_tokenizer is None:
+         raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Model is not loaded.")
+    model_config = global_model.config.to_dict()
+    tokenizer_config = global_tokenizer.init_kwargs
+    special_tokens = {
+        "eos_token": global_tokenizer.eos_token,
+        "eos_token_id": global_tokenizer.eos_token_id,
+        "pad_token": global_tokenizer.pad_token,
+        "pad_token_id": global_tokenizer.pad_token_id,
+        "bos_token": global_tokenizer.bos_token,
+        "bos_token_id": global_tokenizer.bos_token_id,
+        "unk_token": global_tokenizer.unk_token,
+        "unk_token_id": global_tokenizer.unk_token_id,
+    }
+    return JSONResponse({
+        "model_name": MODEL_NAME,
+        "model_config": model_config,
+        "tokenizer_config": tokenizer_config,
+        "special_tokens": special_tokens,
+        "vocab_size": global_tokenizer.vocab_size,
+        "max_model_length": global_tokenizer.model_max_length,
+        "is_loaded": True,
+    })
+
+@app.get("/tokenizer/chat_template", summary="Get the model's chat template", dependencies=[Depends(authenticate_api_key)])
+async def get_chat_template():
+    if global_tokenizer is None:
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Tokenizer is not loaded.")
+    if not hasattr(global_tokenizer, 'chat_template') or not global_tokenizer.chat_template:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tokenizer does not have a chat template.")
+    return JSONResponse({"chat_template": global_tokenizer.chat_template})
+
+
+@app.post("/tokenize", summary="Tokenize text", dependencies=[Depends(authenticate_api_key)])
+async def tokenize_endpoint(req: TokenizeRequest):
+     if global_tokenizer is None:
+          raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Tokenizer is not loaded.")
+     try:
+          text_to_tokenize = req.text
+          if req.chat_format and isinstance(req.text, str):
+               text_to_tokenize = format_conversation(req.text, [], None, global_tokenizer)
+               req.add_special_tokens = False
+
+          tokenizer_kwargs_final = req.tokenizer_kwargs or {}
+          if req.use_fast_tokenizer is not None:
+              tokenizer_kwargs_final['use_fast'] = req.use_fast_tokenizer
+
+          return_tensors = req.return_tensors if req.return_tensors is not None else None
+          if return_tensors is None and (req.return_token_type_ids or req.return_attention_mask or req.return_offsets_mapping or req.return_length or req.return_word_ids):
+               return_tensors = "pt"
+
+          encoded = global_tokenizer(
+               text_to_tokenize,
+               add_special_tokens=req.add_special_tokens,
+               return_token_type_ids=req.return_token_type_ids,
+               padding=req.padding,
+               truncation=req.truncation,
+               max_length=req.max_length,
+               is_split_into_words=req.is_split_into_words,
+               return_tensors=return_tensors,
+               return_attention_mask=req.return_attention_mask,
+               return_offsets_mapping=req.return_offsets_mapping,
+               return_length=req.return_length,
+               return_word_ids=req.return_word_ids,
+               verbose=req.verbose,
+               **tokenizer_kwargs_final
+          )
+          if return_tensors:
+               return JSONResponse({k: v.tolist() if isinstance(v, torch.Tensor) else v for k, v in encoded.items()})
+          else:
+               if isinstance(req.text, str):
+                    response_data = {
+                         "tokens": encoded["input_ids"],
+                         "token_count": len(encoded["input_ids"]),
+                         "input_text": req.text
+                         }
+                    if req.chat_format:
+                         response_data["formatted_text"] = text_to_tokenize
+                    if "token_type_ids" in encoded: response_data["token_type_ids"] = encoded["token_type_ids"]
+                    if "attention_mask" in encoded: response_data["attention_mask"] = encoded["attention_mask"]
+                    if "offset_mapping" in encoded: response_data["offset_mapping"] = encoded["offset_mapping"]
+                    if "length" in encoded: response_data["length"] = encoded["length"]
+                    if "word_ids" in encoded: response_data["word_ids"] = encoded["word_ids"]
+                    return JSONResponse(response_data)
+               elif isinstance(req.text, list):
+                    results = []
+                    for i, item_text in enumerate(req.text):
+                         item_result: Dict[str, Any] = {"text": item_text}
+                         if i < len(encoded.get("input_ids", [])):
+                              item_result["tokens"] = encoded["input_ids"][i]
+                              item_result["token_count"] = len(encoded["input_ids"][i])
+                              if "token_type_ids" in encoded and i < len(encoded["token_type_ids"]):
+                                   item_result["token_type_ids"] = encoded["token_type_ids"][i]
+                              if "attention_mask" in encoded and i < len(encoded["attention_mask"]):
+                                   item_result["attention_mask"] = encoded["attention_mask"][i]
+                              if "offset_mapping" in encoded and i < len(encoded["offset_mapping"]):
+                                   item_result["offset_mapping"] = encoded["offset_mapping"][i]
+                              if "length" in encoded and i < len(encoded["length"]):
+                                   item_result["length"] = encoded["length"][i]
+                              if "word_ids" in encoded and i < len(encoded["word_ids"]):
+                                   item_result["word_ids"] = encoded["word_ids"][i]
+                         else:
+                              item_result["tokens"] = []
+                              item_result["token_count"] = 0
+                         results.append(item_result)
+                    return JSONResponse(results)
+     except Exception as e:
+          raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Tokenization error: {e}")
+
+@app.post("/decode", summary="Decode token IDs to text", dependencies=[Depends(authenticate_api_key)])
+async def decode_endpoint(req: DecodeRequest):
+     if global_tokenizer is None:
+          raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Tokenizer is not loaded.")
+     try:
+          decode_kwargs = req.decode_kwargs or {}
+          text = global_tokenizer.decode(
+              req.token_ids,
+              skip_special_tokens=req.skip_special_tokens,
+              clean_up_tokenization_spaces=req.clean_up_tokenization_spaces,
+              **decode_kwargs
+          )
+          return JSONResponse({"text": text})
+     except Exception as e:
+          raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Decoding error: {e}")
+
+@app.post("/system_prompt", summary="Update global system prompt", dependencies=[Depends(authenticate_api_key)])
+async def update_system_prompt(req: SystemPromptUpdateRequest):
+     global SYSTEM_PROMPT
+     if req.system_prompt is not None:
+          SYSTEM_PROMPT = req.system_prompt.strip()
+          return JSONResponse({"status": "success", "message": "Global system prompt updated"})
+     else:
+          raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="System prompt cannot be null")
+
+@app.get("/system_prompt", summary="Get global system prompt", dependencies=[Depends(authenticate_api_key)])
+async def get_system_prompt():
+    return JSONResponse({"system_prompt": SYSTEM_PROMPT})
+
+@app.post("/model/reload", summary="Reload the model", dependencies=[Depends(authenticate_api_key)])
+async def reload_model(req: ModelReloadRequest):
+     global global_model, global_tokenizer, global_tokens, MODEL_NAME, TRUST_REMOTE_CODE, ENABLE_FLASH_ATTENTION_2, TORCH_DTYPE, TORCH_DTYPE_STR
+     new_model_name = req.model_name if req.model_name else MODEL_NAME
+     new_trust_remote_code = req.trust_remote_code if req.trust_remote_code is not None else TRUST_REMOTE_CODE
+     new_enable_flash_attention_2 = req.enable_flash_attention_2 if req.enable_flash_attention_2 is not None else ENABLE_FLASH_ATTENTION_2
+     new_torch_dtype_str_req = req.torch_dtype if req.torch_dtype else TORCH_DTYPE_STR
+     try:
+          new_torch_dtype = getattr(torch, new_torch_dtype_str_req.lower())
+          if new_torch_dtype_str_req.lower() == "bfloat16" and (not torch.cuda.is_available() or (torch.cuda.is_available() and not torch.cuda.get_device_properties(0).has_bfloat16)):
+               new_torch_dtype = torch.float16
+          elif not isinstance(new_torch_dtype, torch.dtype):
+               raise AttributeError
+          new_torch_dtype_str = str(new_torch_dtype).split('.')[-1]
+     except AttributeError:
+          raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Invalid or unsupported torch_dtype: {new_torch_dtype_str_req}")
+     device = "cuda" if torch.cuda.is_available() else "cpu"
+     async def _reload_model():
+          global global_model, global_tokenizer, global_tokens, MODEL_NAME, TRUST_REMOTE_CODE, ENABLE_FLASH_ATTENTION_2, TORCH_DTYPE, TORCH_DTYPE_STR
+          try:
+               await cleanup(device)
+               if global_model is not None:
+                    del global_model
+                    global_model = None
+               if global_tokenizer is not None:
+                    del global_tokenizer
+                    global_tokenizer = None
+               global_tokens = {}
+               torch.cuda.empty_cache()
+               gc.collect()
+               config = AutoConfig.from_pretrained(new_model_name, trust_remote_code=new_trust_remote_code)
+               tokenizer_kwargs = {"config": config, "trust_remote_code": new_trust_remote_code}
+               if req.tokenizer_kwargs:
+                    tokenizer_kwargs.update(req.tokenizer_kwargs)
+               tokenizer = AutoTokenizer.from_pretrained(new_model_name, **tokenizer_kwargs)
+               model_kwargs = {"config": config, "torch_dtype": new_torch_dtype, "trust_remote_code": new_trust_remote_code}
+               if new_enable_flash_attention_2 and device == "cuda":
+                    try:
+                         if hasattr(config, '_supports_flash_attn_2') and config._supports_flash_attn_2:
+                             model_kwargs['use_flash_attention_2'] = True
+                         elif hasattr(AutoModelForCausalLM, '_supports_flash_attn_2') and AutoModelForCausalLM._supports_flash_attn_2(config):
+                              model_kwargs['use_flash_attention_2'] = True
+                    except Exception:
+                         pass
+               if req.model_kwargs:
+                    model_kwargs.update(req.model_kwargs)
+               model = AutoModelForCausalLM.from_pretrained(new_model_name, **model_kwargs)
+               model.to(device)
+               model.eval()
+               global_model = model
+               global_tokenizer = tokenizer
+               global_tokens["eos_token_id"] = global_tokenizer.eos_token_id
+               global_tokens["pad_token_id"] = global_tokenizer.pad_token_id
+               if global_tokens["pad_token_id"] is None and global_tokens["eos_token_id"] is not None:
+                    global_tokens["pad_token_id"] = global_tokens["eos_token_id"]
+                    if global_model.config.pad_token_id is None:
+                         global_model.config.pad_token_id = global_tokens["pad_token_id"]
+               elif global_tokens["pad_token_id"] is None and global_tokens["eos_token_id"] is None:
+                    pass
+               MODEL_NAME = new_model_name
+               TRUST_REMOTE_CODE = new_trust_remote_code
+               ENABLE_FLASH_ATTENTION_2 = new_enable_flash_attention_2
+               TORCH_DTYPE = new_torch_dtype
+               TORCH_DTYPE_STR = new_torch_dtype_str
+          except Exception as e:
+               global_model = None
+               global_tokenizer = None
+               global_tokens = {}
+     asyncio.create_task(_reload_model())
+     return JSONResponse({"status": "info", "message": f"Attempting to load model {new_model_name} in background. Check logs for status."})
+
+@app.post("/model/unload", summary="Unload the model", dependencies=[Depends(authenticate_api_key)])
+async def unload_model():
+     global global_model, global_tokenizer, global_tokens
+     device = "cuda" if torch.cuda.is_available() else "cpu"
+     try:
+          await cleanup(device)
+          if global_model is not None:
+               del global_model
+               global_model = None
+          if global_tokenizer is not None:
+               del global_tokenizer
+               global_tokenizer = None
+          global_tokens = {}
+          torch.cuda.empty_cache()
+          gc.collect()
+          return JSONResponse({"status": "success", "message": "Model unloaded successfully."})
+     except Exception as e:
+          raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to unload model: {e}")
+
+@app.get("/queue/status", summary="Get current generation queue status", dependencies=[Depends(authenticate_api_key)])
+async def queue_status():
+     try:
+          current_running = MAX_CONCURRENT_GENERATIONS - generation_semaphore._value
+          return JSONResponse({
+              "max_concurrent": MAX_CONCURRENT_GENERATIONS,
+              "currently_running": current_running,
+              "available_slots": generation_semaphore._value
+          })
+     except Exception as e:
+          raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error getting queue status: {e}")
+
+@app.get("/tokens/special", summary="Get special token IDs", dependencies=[Depends(authenticate_api_key)])
+async def get_special_tokens():
+     if global_tokenizer is None:
+          raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Tokenizer is not loaded.")
+     try:
+          return JSONResponse({
+              "eos_token": global_tokenizer.eos_token,
+              "eos_token_id": global_tokenizer.eos_token_id,
+              "pad_token": global_tokenizer.pad_token,
+              "pad_token_id": global_tokenizer.pad_token_id,
+              "bos_token": global_tokenizer.bos_token,
+              "bos_token_id": global_tokenizer.bos_token_id,
+              "unk_token": global_tokenizer.unk_token,
+              "unk_token_id": global_tokenizer.unk_token_id,
+          })
+     except Exception as e:
+         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error getting special tokens: {e}")
+
+@app.post("/generate", summary="Generate text", dependencies=[Depends(authenticate_api_key)])
+async def generate_endpoint(req: GenerateRequest, request: Request):
+    client_host = request.client.host if request.client else "unknown"
+    if client_host in request_counts and request_counts[client_host] >= REQUEST_RATE_LIMIT:
+         raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail="Rate limit exceeded")
+    request_counts[client_host] = request_counts.get(client_host, 0) + 1
+
+    if global_model is None or global_tokenizer is None:
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Model is not loaded. It may still be loading or failed to load.")
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    apply_seed(req.seed)
+    try:
+        initial_prompt_text = format_conversation(req.input_text, req.history, req.system_prompt, global_tokenizer)
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Error formatting conversation: {e}")
+    try:
+        tokenizer_encoding_kwargs = req.tokenizer_kwargs or {}
+        if req.use_fast_tokenizer is not None:
+             tokenizer_encoding_kwargs['use_fast'] = req.use_fast_tokenizer
+        encoded = global_tokenizer(initial_prompt_text, return_tensors="pt", add_special_tokens=False, **tokenizer_encoding_kwargs).to(device)
+        initial_ids_before_trunc = encoded.input_ids
+        initial_prompt_tokens_count_before_trunc = initial_ids_before_trunc.shape[-1]
+        ids = truncate_encoded_ids(initial_ids_before_trunc, MAX_CONTEXT_TOKENS)
+    except Exception as e:
+        await cleanup(device)
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Tokenizer encoding error: {e}")
+    if req.tokenize_only:
+         await cleanup(device)
+         return JSONResponse({
+             "prompt_tokens_count": initial_prompt_tokens_count_before_trunc,
+             "max_context_tokens": MAX_CONTEXT_TOKENS,
+             "truncated": initial_prompt_tokens_count_before_trunc > MAX_CONTEXT_TOKENS,
+             "input_text_processed": initial_prompt_text,
+             "input_ids_truncated": ids.tolist()[0]
+         })
+    try:
+        gen_cfg, stopping_criteria, logits_processor, direct_generate_params = get_generation_params(req, ids, global_tokenizer, device)
+    except ValueError as e:
+         await cleanup(device)
+         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Error creating generation parameters: {e}")
+    except Exception as e:
+         await cleanup(device)
+         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Unexpected error getting generation parameters: {e}")
+
+
+    async with generation_semaphore:
+        if req.stream:
+             if req.return_only_text:
+                 pass
+             try:
+                 return StreamingResponse(stream_generation_logic(req, ids, gen_cfg, stopping_criteria, logits_processor, direct_generate_params, device), media_type="application/json")
+             except Exception as e:
+                  await cleanup(device)
+                  raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error setting up streaming generation: {e}")
+        else:
+            try:
+                response_data = await non_stream_generation_logic(req, ids, gen_cfg, stopping_criteria, logits_processor, direct_generate_params, device)
+                if req.return_only_text and response_data.get("generated_sequences"):
+                     if response_data["generated_sequences"]:
+                        return JSONResponse({"text": response_data["generated_sequences"][0].get("text", "")})
+                     else:
+                        return JSONResponse({"text": ""})
+                return JSONResponse(response_data)
+            except HTTPException:
+                 raise
+            except Exception as e:
+                await cleanup(device)
+                raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Non-streaming generation error: {e}")
+
+if __name__ == "__main__":
+    uvicorn.run(
+        app, host="0.0.0.0", port=7860,
+        log_level="critical",
+        access_log=False
+    )