Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /InferiaAI_InferiaLLM /scripts /concurrent_chat_tui.py
| #!/usr/bin/env python3 | |
| """ | |
| Concurrent terminal chat TUI with: | |
| - 3 visible streaming panes | |
| - configurable concurrent request batches | |
| - background processing for extra requests | |
| - realtime stats (TTFT, token speed, request states) | |
| - shared chat context across turns | |
| stdlib-only implementation (curses + http.client + threads). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import curses | |
| import http.client | |
| import json | |
| import os | |
| import queue | |
| import ssl | |
| import textwrap | |
| import threading | |
| import time | |
| from dataclasses import dataclass, field | |
| from typing import Dict, List, Optional | |
| from urllib.parse import urlparse | |
| def now_ms() -> float: | |
| return time.perf_counter() * 1000.0 | |
| def approx_tokens(text: str) -> int: | |
| if not text: | |
| return 0 | |
| return max(1, (len(text.encode("utf-8")) + 3) // 4) | |
| def clip(s: str, max_len: int) -> str: | |
| if max_len <= 0: | |
| return "" | |
| if len(s) <= max_len: | |
| return s | |
| if max_len <= 3: | |
| return s[:max_len] | |
| return s[: max_len - 3] + "..." | |
| class RequestState: | |
| request_id: int | |
| batch_id: int | |
| state: str = "queued" | |
| status_code: int = 0 | |
| started_ms: float = 0.0 | |
| ended_ms: float = 0.0 | |
| ttft_ms: Optional[float] = None | |
| completion_tokens: int = 0 | |
| token_speed_tps: float = 0.0 | |
| text: str = "" | |
| error: str = "" | |
| class BatchState: | |
| batch_id: int | |
| prompt: str | |
| request_ids: List[int] = field(default_factory=list) | |
| started_ms: float = field(default_factory=now_ms) | |
| class ConcurrentChatApp: | |
| def __init__( | |
| self, | |
| url: str, | |
| api_key: str, | |
| model: str, | |
| default_concurrency: int, | |
| max_tokens: int, | |
| temperature: float, | |
| history_turns: int, | |
| system_prompt: str, | |
| max_parallel_connections: int = 100, | |
| ): | |
| self.url = url | |
| self.api_key = api_key | |
| self.model = model | |
| self.default_concurrency = max(1, default_concurrency) | |
| self.max_tokens = max_tokens | |
| self.temperature = temperature | |
| self.history_turns = max(1, history_turns) | |
| self.system_prompt = system_prompt.strip() | |
| parsed = urlparse(self.url) | |
| self.scheme = parsed.scheme or "http" | |
| self.host = parsed.hostname or "localhost" | |
| self.port = parsed.port or (443 if self.scheme == "https" else 80) | |
| self.path = parsed.path or "/v1/chat/completions" | |
| self.ssl_context = ( | |
| ssl.create_default_context() if self.scheme == "https" else None | |
| ) | |
| self.events: queue.Queue[dict] = queue.Queue() | |
| self.requests: Dict[int, RequestState] = {} | |
| self.batches: Dict[int, BatchState] = {} | |
| self.next_request_id = 1 | |
| self.next_batch_id = 1 | |
| self.current_batch_id: Optional[int] = None | |
| self.visible_offset = 0 | |
| self.visible_slots: List[Optional[int]] = [None, None, None] | |
| self.input_buffer = "" | |
| self.notice = "Type a prompt and press Enter. /help for commands." | |
| self.exit_requested = False | |
| # Chat context (without system prompt; appended on request build) | |
| self.context_messages: List[dict] = [] | |
| self.last_turn_batch_id: Optional[int] = None | |
| self.last_turn_assistant_idx: Optional[int] = None | |
| self._lock = threading.Lock() | |
| # Connection pool semaphore to limit concurrent HTTP connections | |
| # This prevents "Too many open files" errors | |
| self._conn_semaphore = threading.Semaphore(max_parallel_connections) | |
| self._max_parallel_connections = max_parallel_connections | |
| def _build_messages(self, prompt: str) -> List[dict]: | |
| # Keep context bounded for predictable latency. | |
| tail = self.context_messages[-(self.history_turns * 2) :] | |
| messages: List[dict] = [] | |
| if self.system_prompt: | |
| messages.append({"role": "system", "content": self.system_prompt}) | |
| messages.extend(tail) | |
| messages.append({"role": "user", "content": prompt}) | |
| return messages | |
| def _current_batch(self) -> Optional[BatchState]: | |
| if self.current_batch_id is None: | |
| return None | |
| return self.batches.get(self.current_batch_id) | |
| def _running_count(self) -> int: | |
| return sum( | |
| 1 for r in self.requests.values() if r.state in ("queued", "running") | |
| ) | |
| def _batch_running_count(self, batch_id: int) -> int: | |
| b = self.batches.get(batch_id) | |
| if not b: | |
| return 0 | |
| return sum( | |
| 1 | |
| for rid in b.request_ids | |
| if rid in self.requests | |
| and self.requests[rid].state in ("queued", "running") | |
| ) | |
| def _set_visible_for_current_batch(self): | |
| batch = self._current_batch() | |
| if not batch: | |
| self.visible_slots = [None, None, None] | |
| return | |
| ids = batch.request_ids | |
| slots = [] | |
| for i in range(3): | |
| idx = self.visible_offset + i | |
| slots.append(ids[idx] if 0 <= idx < len(ids) else None) | |
| self.visible_slots = slots | |
| def _extract_piece(self, event: dict) -> str: | |
| choices = event.get("choices", []) | |
| if not isinstance(choices, list) or not choices: | |
| return "" | |
| first = choices[0] if isinstance(choices[0], dict) else {} | |
| delta = first.get("delta") | |
| if isinstance(delta, dict): | |
| for key in ("content", "reasoning_content", "reasoning"): | |
| value = delta.get(key) | |
| if isinstance(value, str) and value: | |
| return value | |
| message = first.get("message") | |
| if isinstance(message, dict): | |
| value = message.get("content") | |
| if isinstance(value, str): | |
| return value | |
| text = first.get("text") | |
| if isinstance(text, str): | |
| return text | |
| return "" | |
| def _worker_request(self, request_id: int, payload: dict): | |
| """Worker thread that makes HTTP request with connection pooling.""" | |
| started = now_ms() | |
| self.events.put( | |
| {"type": "start", "request_id": request_id, "started_ms": started} | |
| ) | |
| conn = None | |
| acquired = False | |
| try: | |
| # Use semaphore to limit concurrent HTTP connections | |
| # This prevents "Too many open files" errors with high concurrency | |
| acquired = self._conn_semaphore.acquire(timeout=60.0) | |
| if not acquired: | |
| raise Exception("Timeout waiting for available connection slot") | |
| if self.scheme == "https": | |
| conn = http.client.HTTPSConnection( | |
| self.host, | |
| self.port, | |
| timeout=300, | |
| context=self.ssl_context, | |
| ) | |
| else: | |
| conn = http.client.HTTPConnection(self.host, self.port, timeout=300) | |
| raw = json.dumps(payload).encode("utf-8") | |
| headers = { | |
| "Authorization": f"Bearer {self.api_key}", | |
| "Content-Type": "application/json", | |
| "Content-Length": str(len(raw)), | |
| "Connection": "close", | |
| } | |
| conn.request("POST", self.path, body=raw, headers=headers) | |
| response = conn.getresponse() | |
| self.events.put( | |
| { | |
| "type": "response", | |
| "request_id": request_id, | |
| "status_code": int(response.status), | |
| } | |
| ) | |
| if response.status != 200: | |
| body = response.read().decode("utf-8", errors="ignore") | |
| self.events.put( | |
| { | |
| "type": "error", | |
| "request_id": request_id, | |
| "status_code": int(response.status), | |
| "error": clip(body, 500), | |
| "ended_ms": now_ms(), | |
| } | |
| ) | |
| return | |
| first_token_seen = False | |
| first_token_ms = 0.0 | |
| token_count = 0 | |
| text_parts: List[str] = [] | |
| while True: | |
| line = response.readline() | |
| if not line: | |
| break | |
| decoded = line.decode("utf-8", errors="ignore").strip() | |
| if not decoded.startswith("data: "): | |
| continue | |
| data = decoded[6:].strip() | |
| if not data or data == "[DONE]": | |
| continue | |
| try: | |
| event = json.loads(data) | |
| except json.JSONDecodeError: | |
| continue | |
| piece = self._extract_piece(event) | |
| if piece: | |
| if not first_token_seen: | |
| first_token_seen = True | |
| first_token_ms = now_ms() | |
| token_inc = approx_tokens(piece) | |
| token_count += token_inc | |
| text_parts.append(piece) | |
| elapsed_s = max( | |
| (now_ms() - (first_token_ms or started)) / 1000.0, 1e-6 | |
| ) | |
| tps = token_count / elapsed_s | |
| self.events.put( | |
| { | |
| "type": "chunk", | |
| "request_id": request_id, | |
| "piece": piece, | |
| "token_inc": token_inc, | |
| "ttft_ms": (first_token_ms - started) | |
| if first_token_seen | |
| else None, | |
| "token_speed_tps": tps, | |
| } | |
| ) | |
| ended = now_ms() | |
| self.events.put( | |
| { | |
| "type": "done", | |
| "request_id": request_id, | |
| "ended_ms": ended, | |
| "text": "".join(text_parts), | |
| "completion_tokens": token_count, | |
| } | |
| ) | |
| except Exception as e: | |
| self.events.put( | |
| { | |
| "type": "error", | |
| "request_id": request_id, | |
| "status_code": 0, | |
| "error": clip(str(e), 500), | |
| "ended_ms": now_ms(), | |
| } | |
| ) | |
| finally: | |
| try: | |
| if conn is not None: | |
| conn.close() | |
| except Exception: | |
| pass | |
| # Release semaphore after connection is fully closed | |
| if acquired: | |
| self._conn_semaphore.release() | |
| def submit_prompt(self, prompt: str, concurrency: Optional[int] = None): | |
| if self._running_count() > 0: | |
| self.notice = "A batch is still running. Wait, or use /q to quit." | |
| return | |
| prompt = prompt.strip() | |
| if not prompt: | |
| return | |
| n = max(1, concurrency or self.default_concurrency) | |
| batch_id = self.next_batch_id | |
| self.next_batch_id += 1 | |
| payload_base = { | |
| "model": self.model, | |
| "messages": self._build_messages(prompt), | |
| "stream": True, | |
| "temperature": self.temperature, | |
| } | |
| if self.max_tokens > 0: | |
| payload_base["max_tokens"] = self.max_tokens | |
| # Add user turn to context after snapshotting payload messages. | |
| self.context_messages.append({"role": "user", "content": prompt}) | |
| self.last_turn_assistant_idx = None | |
| self.last_turn_batch_id = batch_id | |
| batch = BatchState(batch_id=batch_id, prompt=prompt) | |
| self.batches[batch_id] = batch | |
| self.current_batch_id = batch_id | |
| self.visible_offset = 0 | |
| for _ in range(n): | |
| request_id = self.next_request_id | |
| self.next_request_id += 1 | |
| state = RequestState( | |
| request_id=request_id, batch_id=batch_id, state="queued" | |
| ) | |
| self.requests[request_id] = state | |
| batch.request_ids.append(request_id) | |
| payload = dict(payload_base) | |
| t = threading.Thread( | |
| target=self._worker_request, | |
| args=(request_id, payload), | |
| daemon=True, | |
| ) | |
| t.start() | |
| self._set_visible_for_current_batch() | |
| self.notice = f"Launched batch {batch_id} with {n} requests." | |
| def _maybe_set_context_assistant(self, req: RequestState): | |
| if self.last_turn_batch_id != req.batch_id: | |
| return | |
| if self.last_turn_assistant_idx is None: | |
| self.context_messages.append({"role": "assistant", "content": req.text}) | |
| self.last_turn_assistant_idx = len(self.context_messages) - 1 | |
| def select_context_response(self, request_id: int): | |
| req = self.requests.get(request_id) | |
| if not req: | |
| self.notice = f"Request {request_id} not found." | |
| return | |
| if req.state != "completed" or not req.text: | |
| self.notice = f"Request {request_id} is not completed with text yet." | |
| return | |
| if self.last_turn_batch_id != req.batch_id: | |
| self.notice = f"Request {request_id} is from an older batch." | |
| return | |
| if self.last_turn_assistant_idx is None: | |
| self.context_messages.append({"role": "assistant", "content": req.text}) | |
| self.last_turn_assistant_idx = len(self.context_messages) - 1 | |
| else: | |
| self.context_messages[self.last_turn_assistant_idx]["content"] = req.text | |
| self.notice = f"Context assistant set from request {request_id}." | |
| def process_events(self): | |
| while True: | |
| try: | |
| ev = self.events.get_nowait() | |
| except queue.Empty: | |
| break | |
| req = self.requests.get(ev.get("request_id")) | |
| if not req: | |
| continue | |
| ev_type = ev.get("type") | |
| if ev_type == "start": | |
| req.state = "running" | |
| req.started_ms = float(ev.get("started_ms") or now_ms()) | |
| elif ev_type == "response": | |
| req.status_code = int(ev.get("status_code") or 0) | |
| elif ev_type == "chunk": | |
| piece = ev.get("piece") or "" | |
| req.text += piece | |
| req.completion_tokens += int(ev.get("token_inc") or 0) | |
| if req.ttft_ms is None and ev.get("ttft_ms") is not None: | |
| req.ttft_ms = float(ev.get("ttft_ms")) | |
| req.token_speed_tps = float(ev.get("token_speed_tps") or 0.0) | |
| elif ev_type == "done": | |
| req.state = "completed" | |
| req.ended_ms = float(ev.get("ended_ms") or now_ms()) | |
| req.status_code = req.status_code or 200 | |
| if not req.text: | |
| req.text = str(ev.get("text") or "") | |
| req.completion_tokens = max( | |
| req.completion_tokens, int(ev.get("completion_tokens") or 0) | |
| ) | |
| if req.ttft_ms is None: | |
| req.ttft_ms = req.ended_ms - req.started_ms | |
| duration_from_ttft_s = max( | |
| ((req.ended_ms - req.started_ms) - (req.ttft_ms or 0.0)) / 1000.0, | |
| 1e-6, | |
| ) | |
| req.token_speed_tps = req.completion_tokens / duration_from_ttft_s | |
| if req.text: | |
| self._maybe_set_context_assistant(req) | |
| elif ev_type == "error": | |
| req.state = "error" | |
| req.ended_ms = float(ev.get("ended_ms") or now_ms()) | |
| req.status_code = int(ev.get("status_code") or 0) | |
| req.error = str(ev.get("error") or "unknown error") | |
| if req.ttft_ms is None and req.started_ms > 0: | |
| req.ttft_ms = req.ended_ms - req.started_ms | |
| # Auto-close batch when all requests done | |
| batch = self._current_batch() | |
| if batch and self._batch_running_count(batch.batch_id) == 0: | |
| self.notice = f"Batch {batch.batch_id} completed. Use /use <id> to set context response." | |
| def _draw_box(self, stdscr, y: int, x: int, h: int, w: int): | |
| if h < 2 or w < 2: | |
| return | |
| self._safe_addstr(stdscr, y, x, "+" + "-" * (w - 2) + "+") | |
| for row in range(y + 1, y + h - 1): | |
| self._safe_addstr(stdscr, row, x, "|") | |
| self._safe_addstr(stdscr, row, x + w - 1, "|") | |
| self._safe_addstr(stdscr, y + h - 1, x, "+" + "-" * (w - 2) + "+") | |
| def _safe_addstr(self, stdscr, y: int, x: int, text: str, attr: int = 0): | |
| if text is None: | |
| return | |
| max_y, max_x = stdscr.getmaxyx() | |
| if y < 0 or y >= max_y: | |
| return | |
| # Avoid writing into the terminal's final column to prevent curses ERR on edge writes. | |
| if x < 0 or x >= max_x - 1: | |
| return | |
| max_chars = max_x - x - 1 | |
| if max_chars <= 0: | |
| return | |
| out = text[:max_chars] | |
| if not out: | |
| return | |
| try: | |
| if attr: | |
| stdscr.addstr(y, x, out, attr) | |
| else: | |
| stdscr.addstr(y, x, out) | |
| except curses.error: | |
| return | |
| def _safe_move(self, stdscr, y: int, x: int): | |
| max_y, max_x = stdscr.getmaxyx() | |
| if max_y <= 0 or max_x <= 1: | |
| return | |
| y = max(0, min(max_y - 1, y)) | |
| x = max(0, min(max_x - 2, x)) | |
| try: | |
| stdscr.move(y, x) | |
| except curses.error: | |
| return | |
| def _pane_lines( | |
| self, req: Optional[RequestState], width: int, height: int | |
| ) -> List[str]: | |
| if width <= 1 or height <= 0: | |
| return [] | |
| if req is None: | |
| return ["(empty)"][:height] | |
| if req.ttft_ms is not None: | |
| header = f"#{req.request_id} {req.state} sc={req.status_code} ttft={req.ttft_ms:.1f}ms " | |
| else: | |
| header = f"#{req.request_id} {req.state} sc={req.status_code} " | |
| tps = f"tps={req.token_speed_tps:.1f} tok={req.completion_tokens}" | |
| lines = [clip(header + tps, width)] | |
| body = ( | |
| req.text | |
| if req.text | |
| else (req.error if req.error else "(waiting for chunks...)") | |
| ) | |
| wrapped: List[str] = [] | |
| for part in body.splitlines() or [""]: | |
| wrapped.extend(textwrap.wrap(part, width=width) or [""]) | |
| if len(wrapped) > max(0, height - 1): | |
| wrapped = wrapped[-(height - 1) :] | |
| lines.extend(wrapped) | |
| return lines[:height] | |
| def _draw(self, stdscr): | |
| stdscr.erase() | |
| h, w = stdscr.getmaxyx() | |
| if h < 20 or w < 90: | |
| self._safe_addstr( | |
| stdscr, 0, 0, "Terminal too small. Please resize to at least 90x20." | |
| ) | |
| stdscr.refresh() | |
| return | |
| title = "Concurrent Chat TUI (3 visible panes, background concurrency)" | |
| self._safe_addstr(stdscr, 0, 0, clip(title, w - 1), curses.A_BOLD) | |
| self._safe_addstr( | |
| stdscr, | |
| 1, | |
| 0, | |
| clip( | |
| f"URL={self.url} model={self.model} default_concurrency={self.default_concurrency} running={self._running_count()} max_conn={self._max_parallel_connections}", | |
| w - 1, | |
| ), | |
| ) | |
| pane_top = 2 | |
| input_h = 3 | |
| status_h = 8 | |
| pane_h = h - pane_top - status_h - input_h | |
| pane_w = (w - 2) // 3 | |
| gap = 1 | |
| # Draw 3 panes | |
| for i in range(3): | |
| x = i * (pane_w + gap) | |
| self._draw_box(stdscr, pane_top, x, pane_h, pane_w) | |
| rid = self.visible_slots[i] | |
| req = self.requests.get(rid) if rid is not None else None | |
| pane_title = f" Pane {i + 1} " | |
| if rid is not None: | |
| pane_title += f"(req {rid}) " | |
| self._safe_addstr( | |
| stdscr, pane_top, x + 2, clip(pane_title, pane_w - 4), curses.A_BOLD | |
| ) | |
| lines = self._pane_lines(req, pane_w - 2, pane_h - 2) | |
| for idx, line in enumerate(lines): | |
| row = pane_top + 1 + idx | |
| if row >= pane_top + pane_h - 1: | |
| break | |
| self._safe_addstr(stdscr, row, x + 1, clip(line, pane_w - 2)) | |
| status_top = pane_top + pane_h | |
| self._draw_box(stdscr, status_top, 0, status_h, w) | |
| total = len(self.requests) | |
| queued = sum(1 for r in self.requests.values() if r.state == "queued") | |
| running = sum(1 for r in self.requests.values() if r.state == "running") | |
| completed = sum(1 for r in self.requests.values() if r.state == "completed") | |
| errors = sum(1 for r in self.requests.values() if r.state == "error") | |
| self._safe_addstr( | |
| stdscr, | |
| status_top + 1, | |
| 2, | |
| clip( | |
| f"requests total={total} queued={queued} running={running} completed={completed} error={errors}", | |
| w - 4, | |
| ), | |
| ) | |
| batch = self._current_batch() | |
| if batch: | |
| running_in_batch = self._batch_running_count(batch.batch_id) | |
| elapsed = max((now_ms() - batch.started_ms) / 1000.0, 1e-6) | |
| done_in_batch = len( | |
| [ | |
| rid | |
| for rid in batch.request_ids | |
| if self.requests[rid].state in ("completed", "error") | |
| ] | |
| ) | |
| rps = done_in_batch / elapsed | |
| self._safe_addstr( | |
| stdscr, | |
| status_top + 2, | |
| 2, | |
| clip( | |
| f"active batch={batch.batch_id} size={len(batch.request_ids)} done={done_in_batch} running={running_in_batch} " | |
| f"batch_rps={rps:.2f} visible_offset={self.visible_offset}", | |
| w - 4, | |
| ), | |
| ) | |
| self._safe_addstr( | |
| stdscr, | |
| status_top + 3, | |
| 2, | |
| clip(f"prompt: {batch.prompt}", w - 4), | |
| ) | |
| else: | |
| self._safe_addstr(stdscr, status_top + 2, 2, "no active batch") | |
| active_ids = sorted( | |
| [ | |
| r.request_id | |
| for r in self.requests.values() | |
| if r.state in ("queued", "running") | |
| ] | |
| )[:12] | |
| self._safe_addstr( | |
| stdscr, | |
| status_top + 4, | |
| 2, | |
| clip(f"in-flight request IDs: {active_ids if active_ids else '[]'}", w - 4), | |
| ) | |
| self._safe_addstr( | |
| stdscr, | |
| status_top + 5, | |
| 2, | |
| clip( | |
| "commands: /n <num> /use <request_id> /clear /help /q | keys: [ and ] shift visible panes", | |
| w - 4, | |
| ), | |
| ) | |
| self._safe_addstr( | |
| stdscr, status_top + 6, 2, clip(f"notice: {self.notice}", w - 4) | |
| ) | |
| input_top = status_top + status_h | |
| self._draw_box(stdscr, input_top, 0, input_h, w) | |
| self._safe_addstr(stdscr, input_top + 1, 2, "prompt> ") | |
| max_input = max(0, w - 12) | |
| shown = clip(self.input_buffer, max_input) | |
| self._safe_addstr(stdscr, input_top + 1, 10, shown) | |
| self._safe_move(stdscr, input_top + 1, 10 + len(shown)) | |
| stdscr.refresh() | |
| def _handle_command(self, raw: str): | |
| parts = raw.strip().split() | |
| if not parts: | |
| return | |
| cmd = parts[0].lower() | |
| if cmd in ("/q", "/quit", "/exit"): | |
| self.exit_requested = True | |
| return | |
| if cmd == "/help": | |
| self.notice = ( | |
| "Enter text to launch a batch. Commands: /n <num>, /use <id>, /clear, /q. " | |
| "Use [ and ] to shift visible panes." | |
| ) | |
| return | |
| if cmd == "/clear": | |
| if self._running_count() > 0: | |
| self.notice = "Cannot clear while batch is running." | |
| return | |
| self.requests.clear() | |
| self.batches.clear() | |
| self.current_batch_id = None | |
| self.visible_offset = 0 | |
| self.visible_slots = [None, None, None] | |
| self.context_messages.clear() | |
| self.last_turn_assistant_idx = None | |
| self.last_turn_batch_id = None | |
| self.notice = "Cleared requests and context." | |
| return | |
| if cmd == "/n": | |
| if len(parts) != 2: | |
| self.notice = "Usage: /n <concurrency>" | |
| return | |
| try: | |
| n = int(parts[1]) | |
| except ValueError: | |
| self.notice = "Invalid number." | |
| return | |
| if n < 1: | |
| self.notice = "Concurrency must be >= 1." | |
| return | |
| self.default_concurrency = n | |
| self.notice = f"Default concurrency set to {n}." | |
| return | |
| if cmd == "/use": | |
| if len(parts) != 2: | |
| self.notice = "Usage: /use <request_id>" | |
| return | |
| try: | |
| rid = int(parts[1]) | |
| except ValueError: | |
| self.notice = "Invalid request_id." | |
| return | |
| self.select_context_response(rid) | |
| return | |
| self.notice = f"Unknown command: {cmd}" | |
| def _handle_enter(self): | |
| raw = self.input_buffer.strip() | |
| self.input_buffer = "" | |
| if not raw: | |
| return | |
| if raw.startswith("/"): | |
| self._handle_command(raw) | |
| else: | |
| # Optional per-prompt override: "@24 your prompt" | |
| concurrency_override: Optional[int] = None | |
| if raw.startswith("@") and " " in raw: | |
| head, rest = raw.split(" ", 1) | |
| if head[1:].isdigit(): | |
| concurrency_override = max(1, int(head[1:])) | |
| raw = rest.strip() | |
| self.submit_prompt(raw, concurrency_override) | |
| def run(self, stdscr): | |
| curses.curs_set(1) | |
| stdscr.nodelay(True) | |
| stdscr.timeout(80) | |
| while not self.exit_requested: | |
| self.process_events() | |
| self._set_visible_for_current_batch() | |
| self._draw(stdscr) | |
| ch = stdscr.getch() | |
| if ch == -1: | |
| continue | |
| if ch in (curses.KEY_ENTER, 10, 13): | |
| self._handle_enter() | |
| continue | |
| if ch in (curses.KEY_BACKSPACE, 127, 8): | |
| self.input_buffer = self.input_buffer[:-1] | |
| continue | |
| if ch == 9: | |
| self.input_buffer += " " | |
| continue | |
| if ch == 27: | |
| # ESC | |
| self.exit_requested = True | |
| continue | |
| # Pane shift keys when input line is empty | |
| if not self.input_buffer and ch == ord("["): | |
| self.visible_offset = max(0, self.visible_offset - 1) | |
| self._set_visible_for_current_batch() | |
| continue | |
| if not self.input_buffer and ch == ord("]"): | |
| batch = self._current_batch() | |
| max_offset = 0 | |
| if batch: | |
| max_offset = max(0, len(batch.request_ids) - 3) | |
| self.visible_offset = min(max_offset, self.visible_offset + 1) | |
| self._set_visible_for_current_batch() | |
| continue | |
| if 32 <= ch <= 126: | |
| self.input_buffer += chr(ch) | |
| def parse_args(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument( | |
| "--url", | |
| default=os.getenv("CHAT_API_URL", "http://localhost:8001/v1/chat/completions"), | |
| ) | |
| parser.add_argument( | |
| "--api-key", | |
| default=os.getenv("CHAT_API_KEY", ""), | |
| ) | |
| parser.add_argument( | |
| "--model", | |
| default=os.getenv("CHAT_MODEL", "openai/gpt-oss-20b"), | |
| ) | |
| parser.add_argument("--concurrency", type=int, default=12) | |
| parser.add_argument("--max-tokens", type=int, default=2560) | |
| parser.add_argument("--temperature", type=float, default=0.2) | |
| parser.add_argument("--history-turns", type=int, default=8) | |
| parser.add_argument("--system-prompt", default=os.getenv("CHAT_SYSTEM_PROMPT", "")) | |
| parser.add_argument( | |
| "--max-parallel-connections", | |
| type=int, | |
| default=100, | |
| help="Max concurrent HTTP connections (prevents 'Too many open files')", | |
| ) | |
| return parser.parse_args() | |
| def main(): | |
| args = parse_args() | |
| if not args.api_key: | |
| raise SystemExit("Missing API key. Pass --api-key or set CHAT_API_KEY in env.") | |
| app = ConcurrentChatApp( | |
| url=args.url, | |
| api_key=args.api_key, | |
| model=args.model, | |
| default_concurrency=args.concurrency, | |
| max_tokens=args.max_tokens, | |
| temperature=args.temperature, | |
| history_turns=args.history_turns, | |
| system_prompt=args.system_prompt, | |
| max_parallel_connections=args.max_parallel_connections, | |
| ) | |
| curses.wrapper(app.run) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 29.2 kB
- Xet hash:
- 32ae94e1f6dbfab876e4945e928ecd32ca18e845bec30db09ffb0a38da776ee1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.