Spaces:
Sleeping
Sleeping
Dmitry Beresnev commited on
Commit ·
6471a06
1
Parent(s): 27c1c3c
fix critical secutity issues
Browse files- .gitignore +3 -0
- Dockerfile +21 -15
- README.md +32 -0
- config.toml.example +34 -2
- cpp/config.cpp +144 -2
- cpp/http_helpers.cpp +113 -3
- cpp/http_helpers.h +13 -1
- cpp/llm_manager.cpp +41 -12
- cpp/llm_manager_types.h +63 -0
- cpp/model_manager.cpp +5 -4
- cpp/request_parsing.cpp +36 -12
- cpp/request_parsing.h +4 -0
- cpp/runtime_components.cpp +43 -10
- cpp/runtime_components.h +4 -1
- cpp/server.cpp +156 -60
- cpp/server.h +22 -16
- cpp/sha256.h +88 -0
.gitignore
CHANGED
|
@@ -72,6 +72,9 @@ cover/
|
|
| 72 |
logs/
|
| 73 |
agi.log
|
| 74 |
|
|
|
|
|
|
|
|
|
|
| 75 |
# Environment variables
|
| 76 |
.env
|
| 77 |
.env.local
|
|
|
|
| 72 |
logs/
|
| 73 |
agi.log
|
| 74 |
|
| 75 |
+
# Runtime config holds API key secrets — never commit it
|
| 76 |
+
config.toml
|
| 77 |
+
|
| 78 |
# Environment variables
|
| 79 |
.env
|
| 80 |
.env.local
|
Dockerfile
CHANGED
|
@@ -18,7 +18,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
| 18 |
# Clone and build llama.cpp with SSL support for HuggingFace Hub
|
| 19 |
WORKDIR /build
|
| 20 |
ARG CACHEBUST=6
|
| 21 |
-
|
|
|
|
|
|
|
| 22 |
#ARG BUILD_PROFILE=fast_build
|
| 23 |
ARG BUILD_PROFILE=fast_runtime
|
| 24 |
ARG BUILD_JOBS=1
|
|
@@ -91,32 +93,22 @@ COPY --from=builder /build/llm-manager /usr/local/bin/llm-manager
|
|
| 91 |
# Update library cache
|
| 92 |
RUN ldconfig
|
| 93 |
|
| 94 |
-
#
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
python3-pip \
|
| 98 |
-
&& rm -rf /var/lib/apt/lists/*
|
| 99 |
-
|
| 100 |
-
# Install Python packages
|
| 101 |
-
RUN pip3 install --no-cache-dir fastapi uvicorn aiohttp pydantic duckduckgo-search beautifulsoup4 lxml --break-system-packages
|
| 102 |
|
| 103 |
# Create non-root user
|
| 104 |
RUN useradd -m -u 1000 user && \
|
| 105 |
mkdir -p /home/user/.cache/llama.cpp && \
|
| 106 |
chown -R user:user /home/user
|
| 107 |
|
| 108 |
-
# Copy application code
|
| 109 |
-
COPY --chown=user:user python/ /home/user/python/
|
| 110 |
-
|
| 111 |
USER user
|
| 112 |
WORKDIR /home/user
|
| 113 |
|
| 114 |
# Set environment variables
|
| 115 |
ENV HOME=/home/user \
|
| 116 |
LLAMA_CACHE=/home/user/.cache/llama.cpp \
|
| 117 |
-
PATH=/home/user/.local/bin:$PATH
|
| 118 |
-
PYTHONPATH=/home/user/python \
|
| 119 |
-
PYTHONUNBUFFERED=1
|
| 120 |
|
| 121 |
EXPOSE 7860
|
| 122 |
|
|
@@ -147,6 +139,20 @@ EXPOSE 7860
|
|
| 147 |
# batches worth multiplying (was 64/32). RAM headroom (~16 GB vs ~5 GB used)
|
| 148 |
# covers the larger compute buffers.
|
| 149 |
# Validate any change with tests/bench.py (see PERFORMANCE_REFACTORING_PLAN.md).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
ENV DEFAULT_MODEL=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
|
| 151 |
MANAGER_HOST=0.0.0.0 \
|
| 152 |
MANAGER_PORT=7860 \
|
|
|
|
| 18 |
# Clone and build llama.cpp with SSL support for HuggingFace Hub
|
| 19 |
WORKDIR /build
|
| 20 |
ARG CACHEBUST=6
|
| 21 |
+
# Pinned release tag (supply chain: never build from a moving branch). Bump
|
| 22 |
+
# deliberately and re-run the endpoint tests.
|
| 23 |
+
ARG LLAMA_CPP_REF=b12367
|
| 24 |
#ARG BUILD_PROFILE=fast_build
|
| 25 |
ARG BUILD_PROFILE=fast_runtime
|
| 26 |
ARG BUILD_JOBS=1
|
|
|
|
| 93 |
# Update library cache
|
| 94 |
RUN ldconfig
|
| 95 |
|
| 96 |
+
# The legacy Python app (python/) is intentionally NOT shipped: it has
|
| 97 |
+
# unauthenticated admin endpoints and a pile of unpinned dependencies. The
|
| 98 |
+
# runtime image contains only llama-server and the C++ manager.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
# Create non-root user
|
| 101 |
RUN useradd -m -u 1000 user && \
|
| 102 |
mkdir -p /home/user/.cache/llama.cpp && \
|
| 103 |
chown -R user:user /home/user
|
| 104 |
|
|
|
|
|
|
|
|
|
|
| 105 |
USER user
|
| 106 |
WORKDIR /home/user
|
| 107 |
|
| 108 |
# Set environment variables
|
| 109 |
ENV HOME=/home/user \
|
| 110 |
LLAMA_CACHE=/home/user/.cache/llama.cpp \
|
| 111 |
+
PATH=/home/user/.local/bin:$PATH
|
|
|
|
|
|
|
| 112 |
|
| 113 |
EXPOSE 7860
|
| 114 |
|
|
|
|
| 139 |
# batches worth multiplying (was 64/32). RAM headroom (~16 GB vs ~5 GB used)
|
| 140 |
# covers the larger compute buffers.
|
| 141 |
# Validate any change with tests/bench.py (see PERFORMANCE_REFACTORING_PLAN.md).
|
| 142 |
+
#
|
| 143 |
+
# Security (see docs/SECURITY_AUDIT_AND_FIX_PLAN.md):
|
| 144 |
+
# - API keys are NOT baked into the image. Set the API_KEYS secret on the
|
| 145 |
+
# Space: API_KEYS="admin-main:admin:<32+ char secret>,user-main:user:<secret>"
|
| 146 |
+
# (a secret may also be given as sha256:<hex>). Without it the manager
|
| 147 |
+
# fails closed and every authenticated endpoint returns 401.
|
| 148 |
+
# - ALLOWED_MODELS restricts /switch-model; the default model is always allowed.
|
| 149 |
+
# - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
|
| 150 |
+
# (X-Forwarded-For from the HF proxy).
|
| 151 |
+
ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
|
| 152 |
+
PER_IP_REQUESTS_PER_MINUTE=120 \
|
| 153 |
+
MANAGER_MAX_SESSIONS=256 \
|
| 154 |
+
MAX_REQUEST_BODY_BYTES=1048576 \
|
| 155 |
+
LOG_REQUEST_BODIES=false
|
| 156 |
ENV DEFAULT_MODEL=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
|
| 157 |
MANAGER_HOST=0.0.0.0 \
|
| 158 |
MANAGER_PORT=7860 \
|
README.md
CHANGED
|
@@ -25,6 +25,38 @@ The current primary path in this repository is the C++ `llm-manager`:
|
|
| 25 |
|
| 26 |
The legacy Python application is still present under [`python/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/python), but the repository is now centered around the C++ manager implementation in [`cpp/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/cpp).
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
## Features
|
| 29 |
|
| 30 |
- Dynamic model switching through a single manager endpoint.
|
|
|
|
| 25 |
|
| 26 |
The legacy Python application is still present under [`python/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/python), but the repository is now centered around the C++ manager implementation in [`cpp/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/cpp).
|
| 27 |
|
| 28 |
+
## Security Setup (read before deploying)
|
| 29 |
+
|
| 30 |
+
The manager **fails closed**: with no usable API keys configured every authenticated endpoint
|
| 31 |
+
(`/v1/chat/completions`, `/switch-model`, `/stop`, cancel, `/props`, `/slots`, `/metrics`) returns 401.
|
| 32 |
+
Keys are never baked into the image. Configure them as an environment variable / HF Space secret:
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
# key_id:role:secret[,key_id:role:secret...]; role is admin or user
|
| 36 |
+
API_KEYS="admin-main:admin:$(openssl rand -hex 24),user-main:user:$(openssl rand -hex 24)"
|
| 37 |
+
# a secret may also be given as sha256:<hex> so the plaintext is never stored
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
Rules enforced at startup (violations disable the key and are logged):
|
| 41 |
+
|
| 42 |
+
- secrets must be at least 32 characters (`AUTH_MIN_SECRET_LENGTH`)
|
| 43 |
+
- placeholder secrets (`change-me…`, `test…`, `replace…`, …) are refused
|
| 44 |
+
- `AUTH_ALLOW_ANONYMOUS=true` lets unauthenticated callers in with the **user** role only
|
| 45 |
+
|
| 46 |
+
Other security-relevant settings (all overridable by environment variable):
|
| 47 |
+
|
| 48 |
+
| Setting | Env | Purpose |
|
| 49 |
+
|---------|-----|---------|
|
| 50 |
+
| `[models] allowed` | `ALLOWED_MODELS` | Allowlist for `/switch-model`; default model always allowed |
|
| 51 |
+
| `[rate_limit] per_ip_requests_per_minute` | `PER_IP_REQUESTS_PER_MINUTE` | Pre-auth per-address limit (uses `X-Forwarded-For` when `trust_proxy_headers`) |
|
| 52 |
+
| `[server] max_sessions` | `MANAGER_MAX_SESSIONS` | Connection cap; excess gets a fast 503 |
|
| 53 |
+
| `[limits] max_request_body_bytes` | `MAX_REQUEST_BODY_BYTES` | 413 for oversized bodies |
|
| 54 |
+
| `[logging] log_request_bodies` | `LOG_REQUEST_BODIES` | Prompts stay out of logs unless enabled |
|
| 55 |
+
|
| 56 |
+
Cancellation is soft: cancelling a running request closes only that request's upstream connection.
|
| 57 |
+
Only the admin `/stop` endpoint restarts the worker. The full audit and rationale is in
|
| 58 |
+
`docs/SECURITY_AUDIT_AND_FIX_PLAN.md`.
|
| 59 |
+
|
| 60 |
## Features
|
| 61 |
|
| 62 |
- Dynamic model switching through a single manager endpoint.
|
config.toml.example
CHANGED
|
@@ -1,6 +1,10 @@
|
|
| 1 |
[server]
|
| 2 |
host = "0.0.0.0"
|
| 3 |
port = 7860
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
[worker]
|
| 6 |
default_model = "QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"
|
|
@@ -25,11 +29,18 @@ ubatch = 128
|
|
| 25 |
[auth]
|
| 26 |
header = "Authorization"
|
| 27 |
scheme = "Bearer"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
[limits]
|
| 30 |
default_max_tokens = 256
|
| 31 |
max_tokens_per_request = 2048
|
| 32 |
request_timeout_sec = 30
|
|
|
|
|
|
|
| 33 |
|
| 34 |
[queue]
|
| 35 |
max_size = 100
|
|
@@ -46,15 +57,36 @@ enabled = false
|
|
| 46 |
[rate_limit]
|
| 47 |
requests_per_minute = 60
|
| 48 |
estimated_tokens_per_minute = 6000
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
[[api_keys]]
|
| 51 |
key_id = "admin-main"
|
| 52 |
-
|
|
|
|
| 53 |
role = "admin"
|
| 54 |
enabled = true
|
| 55 |
|
| 56 |
[[api_keys]]
|
| 57 |
key_id = "user-main"
|
| 58 |
-
|
|
|
|
| 59 |
role = "user"
|
| 60 |
enabled = true
|
|
|
|
| 1 |
[server]
|
| 2 |
host = "0.0.0.0"
|
| 3 |
port = 7860
|
| 4 |
+
# Max concurrently open client connections (each pins a thread); excess get 503.
|
| 5 |
+
max_sessions = 256
|
| 6 |
+
# Per-recv timeout for client sockets (idle keep-alive, slow senders).
|
| 7 |
+
read_timeout_sec = 30
|
| 8 |
|
| 9 |
[worker]
|
| 10 |
default_model = "QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"
|
|
|
|
| 29 |
[auth]
|
| 30 |
header = "Authorization"
|
| 31 |
scheme = "Bearer"
|
| 32 |
+
# Fail closed: with no usable api keys every authenticated endpoint is 401.
|
| 33 |
+
# Set true to let unauthenticated callers in with the "user" role (never admin).
|
| 34 |
+
allow_anonymous = false
|
| 35 |
+
# Plaintext secrets shorter than this are refused at load time.
|
| 36 |
+
min_secret_length = 32
|
| 37 |
|
| 38 |
[limits]
|
| 39 |
default_max_tokens = 256
|
| 40 |
max_tokens_per_request = 2048
|
| 41 |
request_timeout_sec = 30
|
| 42 |
+
# Bodies above this get 413 before parsing.
|
| 43 |
+
max_request_body_bytes = 1048576
|
| 44 |
|
| 45 |
[queue]
|
| 46 |
max_size = 100
|
|
|
|
| 57 |
[rate_limit]
|
| 58 |
requests_per_minute = 60
|
| 59 |
estimated_tokens_per_minute = 6000
|
| 60 |
+
# Pre-auth limit per client address, applied to every request (0 = off).
|
| 61 |
+
per_ip_requests_per_minute = 120
|
| 62 |
+
# Use the first X-Forwarded-For hop as the client address (true behind the
|
| 63 |
+
# HF proxy or any trusted reverse proxy; false if clients connect directly).
|
| 64 |
+
trust_proxy_headers = true
|
| 65 |
|
| 66 |
+
[logging]
|
| 67 |
+
# Request bodies contain user prompts; only enable for local debugging.
|
| 68 |
+
log_request_bodies = false
|
| 69 |
+
|
| 70 |
+
[models]
|
| 71 |
+
# Models /switch-model may load (the default model is always allowed).
|
| 72 |
+
allowed = ["QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"]
|
| 73 |
+
|
| 74 |
+
# API keys. Prefer the API_KEYS environment variable (HF Space secret) in
|
| 75 |
+
# production: "key_id:role:secret,key_id:role:secret". When API_KEYS is set
|
| 76 |
+
# the entries below are ignored. Placeholder secrets ("change-me...", "test",
|
| 77 |
+
# ...) and secrets shorter than min_secret_length are disabled at load time.
|
| 78 |
+
# Generate one with: openssl rand -hex 24
|
| 79 |
+
# Hash one with: printf '%s' "$SECRET" | sha256sum
|
| 80 |
[[api_keys]]
|
| 81 |
key_id = "admin-main"
|
| 82 |
+
# Either a plaintext secret ...
|
| 83 |
+
secret = "REPLACE-WITH-32-PLUS-RANDOM-CHARS-admin"
|
| 84 |
role = "admin"
|
| 85 |
enabled = true
|
| 86 |
|
| 87 |
[[api_keys]]
|
| 88 |
key_id = "user-main"
|
| 89 |
+
# ... or only its SHA-256 so the plaintext never sits in the config file.
|
| 90 |
+
secret_sha256 = "0000000000000000000000000000000000000000000000000000000000000000"
|
| 91 |
role = "user"
|
| 92 |
enabled = true
|
cpp/config.cpp
CHANGED
|
@@ -1,12 +1,16 @@
|
|
| 1 |
#include "config.h"
|
| 2 |
|
| 3 |
#include "http_helpers.h"
|
|
|
|
| 4 |
|
| 5 |
#include <algorithm>
|
|
|
|
| 6 |
#include <cstdlib>
|
| 7 |
#include <filesystem>
|
| 8 |
#include <fstream>
|
|
|
|
| 9 |
#include <unordered_map>
|
|
|
|
| 10 |
|
| 11 |
std::string get_env_or(const char *name, const std::string &fallback) {
|
| 12 |
const char *v = std::getenv(name);
|
|
@@ -112,7 +116,7 @@ static std::vector<ApiKeyRecord> parse_api_keys_toml(const std::string &path) {
|
|
| 112 |
bool has_any_field = false;
|
| 113 |
|
| 114 |
auto flush_current = [&]() {
|
| 115 |
-
if (has_any_field && !current.key_id.empty() && !current.secret.empty()) {
|
| 116 |
keys.push_back(current);
|
| 117 |
}
|
| 118 |
current = ApiKeyRecord{};
|
|
@@ -145,6 +149,7 @@ static std::vector<ApiKeyRecord> parse_api_keys_toml(const std::string &path) {
|
|
| 145 |
|
| 146 |
if (key == "key_id") current.key_id = value;
|
| 147 |
else if (key == "secret") current.secret = value;
|
|
|
|
| 148 |
else if (key == "role") current.role = parse_role_or(value, current.role);
|
| 149 |
else if (key == "enabled") current.enabled = parse_bool_or(value, current.enabled);
|
| 150 |
}
|
|
@@ -181,6 +186,112 @@ static int get_toml_int_or(
|
|
| 181 |
}
|
| 182 |
}
|
| 183 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
ManagerConfig load_manager_config() {
|
| 185 |
ManagerConfig cfg;
|
| 186 |
|
|
@@ -195,6 +306,8 @@ ManagerConfig load_manager_config() {
|
|
| 195 |
|
| 196 |
cfg.server.host = get_env_or("MANAGER_HOST", get_toml_string_or(toml, "server", "host", cfg.server.host));
|
| 197 |
cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
|
|
|
|
|
|
|
| 198 |
|
| 199 |
cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
|
| 200 |
cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
|
|
@@ -211,10 +324,15 @@ ManagerConfig load_manager_config() {
|
|
| 211 |
|
| 212 |
cfg.auth.header = get_env_or("AUTH_HEADER", get_toml_string_or(toml, "auth", "header", cfg.auth.header));
|
| 213 |
cfg.auth.scheme = get_env_or("AUTH_SCHEME", get_toml_string_or(toml, "auth", "scheme", cfg.auth.scheme));
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
|
| 215 |
cfg.limits.default_max_tokens = get_env_int_or("DEFAULT_MAX_TOKENS", get_toml_int_or(toml, "limits", "default_max_tokens", cfg.limits.default_max_tokens));
|
| 216 |
cfg.limits.max_tokens_per_request = get_env_int_or("MAX_TOKENS_PER_REQUEST", get_toml_int_or(toml, "limits", "max_tokens_per_request", cfg.limits.max_tokens_per_request));
|
| 217 |
cfg.limits.request_timeout_sec = get_env_int_or("REQUEST_TIMEOUT_SEC", get_toml_int_or(toml, "limits", "request_timeout_sec", cfg.limits.request_timeout_sec));
|
|
|
|
| 218 |
|
| 219 |
cfg.queue.max_size = static_cast<size_t>(std::max(
|
| 220 |
1,
|
|
@@ -225,15 +343,39 @@ ManagerConfig load_manager_config() {
|
|
| 225 |
|
| 226 |
cfg.rate_limit.requests_per_minute = get_env_int_or("REQUESTS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "requests_per_minute", cfg.rate_limit.requests_per_minute));
|
| 227 |
cfg.rate_limit.estimated_tokens_per_minute = get_env_int_or("ESTIMATED_TOKENS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "estimated_tokens_per_minute", cfg.rate_limit.estimated_tokens_per_minute));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
|
| 229 |
cfg.scheduler.max_concurrent = get_env_int_or("SCHEDULER_MAX_CONCURRENT", get_toml_int_or(toml, "scheduler", "max_concurrent", cfg.scheduler.max_concurrent));
|
| 230 |
cfg.streaming.enabled = parse_bool_or(
|
| 231 |
get_env_or("STREAMING_ENABLED", get_toml_string_or(toml, "streaming", "enabled", cfg.streaming.enabled ? "true" : "false")),
|
| 232 |
cfg.streaming.enabled);
|
| 233 |
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
cfg.api_keys = parse_api_keys_toml(config_path);
|
| 236 |
}
|
|
|
|
| 237 |
|
| 238 |
return cfg;
|
| 239 |
}
|
|
|
|
| 1 |
#include "config.h"
|
| 2 |
|
| 3 |
#include "http_helpers.h"
|
| 4 |
+
#include "sha256.h"
|
| 5 |
|
| 6 |
#include <algorithm>
|
| 7 |
+
#include <cctype>
|
| 8 |
#include <cstdlib>
|
| 9 |
#include <filesystem>
|
| 10 |
#include <fstream>
|
| 11 |
+
#include <sstream>
|
| 12 |
#include <unordered_map>
|
| 13 |
+
#include <unordered_set>
|
| 14 |
|
| 15 |
std::string get_env_or(const char *name, const std::string &fallback) {
|
| 16 |
const char *v = std::getenv(name);
|
|
|
|
| 116 |
bool has_any_field = false;
|
| 117 |
|
| 118 |
auto flush_current = [&]() {
|
| 119 |
+
if (has_any_field && !current.key_id.empty() && (!current.secret.empty() || !current.secret_sha256.empty())) {
|
| 120 |
keys.push_back(current);
|
| 121 |
}
|
| 122 |
current = ApiKeyRecord{};
|
|
|
|
| 149 |
|
| 150 |
if (key == "key_id") current.key_id = value;
|
| 151 |
else if (key == "secret") current.secret = value;
|
| 152 |
+
else if (key == "secret_sha256") current.secret_sha256 = value;
|
| 153 |
else if (key == "role") current.role = parse_role_or(value, current.role);
|
| 154 |
else if (key == "enabled") current.enabled = parse_bool_or(value, current.enabled);
|
| 155 |
}
|
|
|
|
| 186 |
}
|
| 187 |
}
|
| 188 |
|
| 189 |
+
// Splits a comma-separated list, trimming and dropping empty entries.
|
| 190 |
+
static std::vector<std::string> split_csv(const std::string &value) {
|
| 191 |
+
std::vector<std::string> out;
|
| 192 |
+
std::stringstream ss(value);
|
| 193 |
+
std::string item;
|
| 194 |
+
while (std::getline(ss, item, ',')) {
|
| 195 |
+
item = strip_quotes(trim_copy(item));
|
| 196 |
+
if (!item.empty()) out.push_back(item);
|
| 197 |
+
}
|
| 198 |
+
return out;
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
// Parses a one-line TOML string array: allowed = ["a", "b"]. The simple
|
| 202 |
+
// parser stores the raw right-hand side, brackets included.
|
| 203 |
+
static std::vector<std::string> parse_toml_string_array(const std::string &raw) {
|
| 204 |
+
std::string inner = trim_copy(raw);
|
| 205 |
+
if (inner.size() >= 2 && inner.front() == '[' && inner.back() == ']') {
|
| 206 |
+
inner = inner.substr(1, inner.size() - 2);
|
| 207 |
+
}
|
| 208 |
+
return split_csv(inner);
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
// API_KEYS="key_id:role:secret,key_id:role:secret". A secret may contain ':'
|
| 212 |
+
// (only the first two separators are significant) and may be given as
|
| 213 |
+
// "sha256:<hex>" to avoid putting the plaintext in the environment.
|
| 214 |
+
static std::vector<ApiKeyRecord> parse_api_keys_env(const std::string &raw) {
|
| 215 |
+
std::vector<ApiKeyRecord> keys;
|
| 216 |
+
for (const auto &entry : split_csv(raw)) {
|
| 217 |
+
const auto first = entry.find(':');
|
| 218 |
+
if (first == std::string::npos) {
|
| 219 |
+
log_line("config: ignoring malformed API_KEYS entry (expected key_id:role:secret)");
|
| 220 |
+
continue;
|
| 221 |
+
}
|
| 222 |
+
const auto second = entry.find(':', first + 1);
|
| 223 |
+
if (second == std::string::npos) {
|
| 224 |
+
log_line("config: ignoring malformed API_KEYS entry (expected key_id:role:secret)");
|
| 225 |
+
continue;
|
| 226 |
+
}
|
| 227 |
+
ApiKeyRecord record;
|
| 228 |
+
record.key_id = trim_copy(entry.substr(0, first));
|
| 229 |
+
record.role = parse_role_or(entry.substr(first + 1, second - first - 1), Role::USER);
|
| 230 |
+
std::string secret = trim_copy(entry.substr(second + 1));
|
| 231 |
+
if (secret.rfind("sha256:", 0) == 0) {
|
| 232 |
+
record.secret_sha256 = secret.substr(7);
|
| 233 |
+
} else {
|
| 234 |
+
record.secret = secret;
|
| 235 |
+
}
|
| 236 |
+
if (record.key_id.empty() || (record.secret.empty() && record.secret_sha256.empty())) {
|
| 237 |
+
log_line("config: ignoring API_KEYS entry with empty key_id or secret");
|
| 238 |
+
continue;
|
| 239 |
+
}
|
| 240 |
+
keys.push_back(record);
|
| 241 |
+
}
|
| 242 |
+
return keys;
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
static bool is_hex_digest(const std::string &value) {
|
| 246 |
+
if (value.size() != 64) return false;
|
| 247 |
+
return std::all_of(value.begin(), value.end(), [](unsigned char c) {
|
| 248 |
+
return std::isxdigit(c) != 0;
|
| 249 |
+
});
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
// Normalises every record to a lowercase hex digest, refuses placeholder and
|
| 253 |
+
// short secrets, and drops duplicates. Refused keys are disabled rather than
|
| 254 |
+
// silently accepted so a misconfigured deployment fails closed.
|
| 255 |
+
static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig &auth) {
|
| 256 |
+
static const char *kPlaceholders[] = {"change-me", "changeme", "replace", "secret", "password", "test", "example"};
|
| 257 |
+
std::unordered_set<std::string> seen_ids;
|
| 258 |
+
for (auto &record : keys) {
|
| 259 |
+
if (!seen_ids.insert(record.key_id).second) {
|
| 260 |
+
log_line("config: duplicate api key id '" + record.key_id + "' — disabling later entry");
|
| 261 |
+
record.enabled = false;
|
| 262 |
+
continue;
|
| 263 |
+
}
|
| 264 |
+
if (!record.secret.empty()) {
|
| 265 |
+
std::string lowered = record.secret;
|
| 266 |
+
std::transform(lowered.begin(), lowered.end(), lowered.begin(), [](unsigned char c) {
|
| 267 |
+
return static_cast<char>(std::tolower(c));
|
| 268 |
+
});
|
| 269 |
+
for (const char *placeholder : kPlaceholders) {
|
| 270 |
+
if (lowered.rfind(placeholder, 0) == 0) {
|
| 271 |
+
log_line("config: api key '" + record.key_id + "' uses a placeholder secret — disabled");
|
| 272 |
+
record.enabled = false;
|
| 273 |
+
break;
|
| 274 |
+
}
|
| 275 |
+
}
|
| 276 |
+
if (record.enabled && static_cast<int>(record.secret.size()) < auth.min_secret_length) {
|
| 277 |
+
log_line("config: api key '" + record.key_id + "' secret shorter than " +
|
| 278 |
+
std::to_string(auth.min_secret_length) + " chars — disabled");
|
| 279 |
+
record.enabled = false;
|
| 280 |
+
}
|
| 281 |
+
record.secret_sha256 = sha256_hex(record.secret);
|
| 282 |
+
// The plaintext is not needed after hashing; drop it from memory.
|
| 283 |
+
record.secret.clear();
|
| 284 |
+
} else {
|
| 285 |
+
std::transform(record.secret_sha256.begin(), record.secret_sha256.end(), record.secret_sha256.begin(),
|
| 286 |
+
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
| 287 |
+
if (!is_hex_digest(record.secret_sha256)) {
|
| 288 |
+
log_line("config: api key '" + record.key_id + "' has an invalid secret_sha256 — disabled");
|
| 289 |
+
record.enabled = false;
|
| 290 |
+
}
|
| 291 |
+
}
|
| 292 |
+
}
|
| 293 |
+
}
|
| 294 |
+
|
| 295 |
ManagerConfig load_manager_config() {
|
| 296 |
ManagerConfig cfg;
|
| 297 |
|
|
|
|
| 306 |
|
| 307 |
cfg.server.host = get_env_or("MANAGER_HOST", get_toml_string_or(toml, "server", "host", cfg.server.host));
|
| 308 |
cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
|
| 309 |
+
cfg.server.max_sessions = std::max(1, get_env_int_or("MANAGER_MAX_SESSIONS", get_toml_int_or(toml, "server", "max_sessions", cfg.server.max_sessions)));
|
| 310 |
+
cfg.server.read_timeout_sec = std::max(1, get_env_int_or("MANAGER_READ_TIMEOUT_SEC", get_toml_int_or(toml, "server", "read_timeout_sec", cfg.server.read_timeout_sec)));
|
| 311 |
|
| 312 |
cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
|
| 313 |
cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
|
|
|
|
| 324 |
|
| 325 |
cfg.auth.header = get_env_or("AUTH_HEADER", get_toml_string_or(toml, "auth", "header", cfg.auth.header));
|
| 326 |
cfg.auth.scheme = get_env_or("AUTH_SCHEME", get_toml_string_or(toml, "auth", "scheme", cfg.auth.scheme));
|
| 327 |
+
cfg.auth.allow_anonymous = parse_bool_or(
|
| 328 |
+
get_env_or("AUTH_ALLOW_ANONYMOUS", get_toml_string_or(toml, "auth", "allow_anonymous", cfg.auth.allow_anonymous ? "true" : "false")),
|
| 329 |
+
cfg.auth.allow_anonymous);
|
| 330 |
+
cfg.auth.min_secret_length = std::max(1, get_env_int_or("AUTH_MIN_SECRET_LENGTH", get_toml_int_or(toml, "auth", "min_secret_length", cfg.auth.min_secret_length)));
|
| 331 |
|
| 332 |
cfg.limits.default_max_tokens = get_env_int_or("DEFAULT_MAX_TOKENS", get_toml_int_or(toml, "limits", "default_max_tokens", cfg.limits.default_max_tokens));
|
| 333 |
cfg.limits.max_tokens_per_request = get_env_int_or("MAX_TOKENS_PER_REQUEST", get_toml_int_or(toml, "limits", "max_tokens_per_request", cfg.limits.max_tokens_per_request));
|
| 334 |
cfg.limits.request_timeout_sec = get_env_int_or("REQUEST_TIMEOUT_SEC", get_toml_int_or(toml, "limits", "request_timeout_sec", cfg.limits.request_timeout_sec));
|
| 335 |
+
cfg.limits.max_request_body_bytes = std::max(1024, get_env_int_or("MAX_REQUEST_BODY_BYTES", get_toml_int_or(toml, "limits", "max_request_body_bytes", cfg.limits.max_request_body_bytes)));
|
| 336 |
|
| 337 |
cfg.queue.max_size = static_cast<size_t>(std::max(
|
| 338 |
1,
|
|
|
|
| 343 |
|
| 344 |
cfg.rate_limit.requests_per_minute = get_env_int_or("REQUESTS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "requests_per_minute", cfg.rate_limit.requests_per_minute));
|
| 345 |
cfg.rate_limit.estimated_tokens_per_minute = get_env_int_or("ESTIMATED_TOKENS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "estimated_tokens_per_minute", cfg.rate_limit.estimated_tokens_per_minute));
|
| 346 |
+
cfg.rate_limit.per_ip_requests_per_minute = get_env_int_or("PER_IP_REQUESTS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "per_ip_requests_per_minute", cfg.rate_limit.per_ip_requests_per_minute));
|
| 347 |
+
cfg.rate_limit.trust_proxy_headers = parse_bool_or(
|
| 348 |
+
get_env_or("TRUST_PROXY_HEADERS", get_toml_string_or(toml, "rate_limit", "trust_proxy_headers", cfg.rate_limit.trust_proxy_headers ? "true" : "false")),
|
| 349 |
+
cfg.rate_limit.trust_proxy_headers);
|
| 350 |
+
cfg.logging.log_request_bodies = parse_bool_or(
|
| 351 |
+
get_env_or("LOG_REQUEST_BODIES", get_toml_string_or(toml, "logging", "log_request_bodies", cfg.logging.log_request_bodies ? "true" : "false")),
|
| 352 |
+
cfg.logging.log_request_bodies);
|
| 353 |
|
| 354 |
cfg.scheduler.max_concurrent = get_env_int_or("SCHEDULER_MAX_CONCURRENT", get_toml_int_or(toml, "scheduler", "max_concurrent", cfg.scheduler.max_concurrent));
|
| 355 |
cfg.streaming.enabled = parse_bool_or(
|
| 356 |
get_env_or("STREAMING_ENABLED", get_toml_string_or(toml, "streaming", "enabled", cfg.streaming.enabled ? "true" : "false")),
|
| 357 |
cfg.streaming.enabled);
|
| 358 |
|
| 359 |
+
// Model allowlist: env overrides file; the default model is always allowed.
|
| 360 |
+
const std::string allowed_env = get_env_or("ALLOWED_MODELS", "");
|
| 361 |
+
if (!allowed_env.empty()) {
|
| 362 |
+
cfg.models.allowed = split_csv(allowed_env);
|
| 363 |
+
} else {
|
| 364 |
+
cfg.models.allowed = parse_toml_string_array(get_toml_string_or(toml, "models", "allowed", ""));
|
| 365 |
+
}
|
| 366 |
+
if (std::find(cfg.models.allowed.begin(), cfg.models.allowed.end(), cfg.worker.default_model) == cfg.models.allowed.end()) {
|
| 367 |
+
cfg.models.allowed.push_back(cfg.worker.default_model);
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
// API keys: the environment (HF Space secrets) takes precedence over the
|
| 371 |
+
// file so a committed config can never override deployed credentials.
|
| 372 |
+
const std::string api_keys_env = get_env_or("API_KEYS", "");
|
| 373 |
+
if (!api_keys_env.empty()) {
|
| 374 |
+
cfg.api_keys = parse_api_keys_env(api_keys_env);
|
| 375 |
+
} else if (!config_path.empty() && std::filesystem::exists(config_path)) {
|
| 376 |
cfg.api_keys = parse_api_keys_toml(config_path);
|
| 377 |
}
|
| 378 |
+
finalize_api_keys(cfg.api_keys, cfg.auth);
|
| 379 |
|
| 380 |
return cfg;
|
| 381 |
}
|
cpp/http_helpers.cpp
CHANGED
|
@@ -4,10 +4,19 @@
|
|
| 4 |
#include <boost/beast/core.hpp>
|
| 5 |
#include <boost/beast/http.hpp>
|
| 6 |
|
|
|
|
|
|
|
| 7 |
#include <ctime>
|
| 8 |
#include <iomanip>
|
| 9 |
#include <iostream>
|
|
|
|
|
|
|
| 10 |
#include <sstream>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
namespace asio = boost::asio;
|
| 13 |
namespace beast = boost::beast;
|
|
@@ -23,7 +32,91 @@ std::string now_utc_iso() {
|
|
| 23 |
}
|
| 24 |
|
| 25 |
void log_line(const std::string &line) {
|
| 26 |
-
std::
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
}
|
| 28 |
|
| 29 |
std::string truncate_body(const std::string &body, size_t max_len) {
|
|
@@ -42,18 +135,34 @@ std::optional<std::string> extract_cancel_request_id(const std::string &path) {
|
|
| 42 |
return request_id;
|
| 43 |
}
|
| 44 |
|
| 45 |
-
std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::
|
| 46 |
asio::io_context ioc;
|
| 47 |
asio::ip::tcp::resolver resolver(ioc);
|
| 48 |
beast::tcp_stream stream(ioc);
|
| 49 |
auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
|
| 50 |
stream.connect(results);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
|
| 52 |
http::request<http::string_body> req{http::verb::post, "/v1/chat/completions", 11};
|
| 53 |
req.set(http::field::host, "127.0.0.1");
|
| 54 |
req.set(http::field::content_type, "application/json");
|
| 55 |
req.set(http::field::user_agent, "llm-manager");
|
| 56 |
-
req.body() =
|
| 57 |
req.prepare_payload();
|
| 58 |
http::write(stream, req);
|
| 59 |
|
|
@@ -71,6 +180,7 @@ ProxiedGetResult forward_get_to_worker(const WorkerInfo &worker, const std::stri
|
|
| 71 |
beast::tcp_stream stream(ioc);
|
| 72 |
auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
|
| 73 |
stream.connect(results);
|
|
|
|
| 74 |
|
| 75 |
http::request<http::string_body> req{http::verb::get, target, 11};
|
| 76 |
req.set(http::field::host, "127.0.0.1");
|
|
|
|
| 4 |
#include <boost/beast/core.hpp>
|
| 5 |
#include <boost/beast/http.hpp>
|
| 6 |
|
| 7 |
+
#include <cctype>
|
| 8 |
+
#include <cstdio>
|
| 9 |
#include <ctime>
|
| 10 |
#include <iomanip>
|
| 11 |
#include <iostream>
|
| 12 |
+
#include <mutex>
|
| 13 |
+
#include <random>
|
| 14 |
#include <sstream>
|
| 15 |
+
#include <stdexcept>
|
| 16 |
+
#include <vector>
|
| 17 |
+
|
| 18 |
+
#include <sys/socket.h>
|
| 19 |
+
#include <sys/time.h>
|
| 20 |
|
| 21 |
namespace asio = boost::asio;
|
| 22 |
namespace beast = boost::beast;
|
|
|
|
| 32 |
}
|
| 33 |
|
| 34 |
void log_line(const std::string &line) {
|
| 35 |
+
std::string safe;
|
| 36 |
+
safe.reserve(line.size());
|
| 37 |
+
for (unsigned char c : line) {
|
| 38 |
+
if (c == '\n') safe += "\\n";
|
| 39 |
+
else if (c == '\r') safe += "\\r";
|
| 40 |
+
else if (c == '\t') safe += "\\t";
|
| 41 |
+
else if (c < 0x20 || c == 0x7f) {
|
| 42 |
+
char buf[8];
|
| 43 |
+
std::snprintf(buf, sizeof(buf), "\\x%02x", c);
|
| 44 |
+
safe += buf;
|
| 45 |
+
} else safe.push_back(static_cast<char>(c));
|
| 46 |
+
}
|
| 47 |
+
std::cout << "[" << now_utc_iso() << "] " << safe << std::endl;
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
void set_socket_timeouts(int fd, int recv_sec, int send_sec) {
|
| 51 |
+
timeval rcv{};
|
| 52 |
+
rcv.tv_sec = recv_sec;
|
| 53 |
+
setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast<const char *>(&rcv), sizeof(rcv));
|
| 54 |
+
timeval snd{};
|
| 55 |
+
snd.tv_sec = send_sec;
|
| 56 |
+
setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char *>(&snd), sizeof(snd));
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
std::string generate_request_id() {
|
| 60 |
+
static std::mutex mu;
|
| 61 |
+
static std::mt19937_64 rng = [] {
|
| 62 |
+
std::random_device rd;
|
| 63 |
+
std::seed_seq seed{rd(), rd(), rd(), rd(), rd(), rd(), rd(), rd()};
|
| 64 |
+
return std::mt19937_64(seed);
|
| 65 |
+
}();
|
| 66 |
+
uint64_t hi, lo;
|
| 67 |
+
{
|
| 68 |
+
std::lock_guard<std::mutex> lock(mu);
|
| 69 |
+
hi = rng();
|
| 70 |
+
lo = rng();
|
| 71 |
+
}
|
| 72 |
+
char buf[33];
|
| 73 |
+
std::snprintf(buf, sizeof(buf), "%016llx%016llx",
|
| 74 |
+
static_cast<unsigned long long>(hi), static_cast<unsigned long long>(lo));
|
| 75 |
+
return std::string(buf);
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
std::optional<std::string> normalize_path(const std::string &raw_path) {
|
| 79 |
+
// Percent-decode first: llama-server (cpp-httplib) decodes before routing,
|
| 80 |
+
// so the manager must compare the same form it will be routed as.
|
| 81 |
+
std::string decoded;
|
| 82 |
+
decoded.reserve(raw_path.size());
|
| 83 |
+
for (size_t i = 0; i < raw_path.size(); ++i) {
|
| 84 |
+
const char c = raw_path[i];
|
| 85 |
+
if (c == '%') {
|
| 86 |
+
if (i + 2 >= raw_path.size() ||
|
| 87 |
+
!std::isxdigit(static_cast<unsigned char>(raw_path[i + 1])) ||
|
| 88 |
+
!std::isxdigit(static_cast<unsigned char>(raw_path[i + 2]))) {
|
| 89 |
+
return std::nullopt;
|
| 90 |
+
}
|
| 91 |
+
decoded.push_back(static_cast<char>(std::stoi(raw_path.substr(i + 1, 2), nullptr, 16)));
|
| 92 |
+
i += 2;
|
| 93 |
+
} else {
|
| 94 |
+
decoded.push_back(c);
|
| 95 |
+
}
|
| 96 |
+
}
|
| 97 |
+
for (unsigned char c : decoded) {
|
| 98 |
+
if (c < 0x20 || c == 0x7f || c == '\\') return std::nullopt;
|
| 99 |
+
}
|
| 100 |
+
if (decoded.empty() || decoded.front() != '/') return std::nullopt;
|
| 101 |
+
|
| 102 |
+
std::vector<std::string> segments;
|
| 103 |
+
std::stringstream ss(decoded);
|
| 104 |
+
std::string seg;
|
| 105 |
+
while (std::getline(ss, seg, '/')) {
|
| 106 |
+
if (seg.empty() || seg == ".") continue;
|
| 107 |
+
if (seg == "..") {
|
| 108 |
+
if (segments.empty()) return std::nullopt;
|
| 109 |
+
segments.pop_back();
|
| 110 |
+
continue;
|
| 111 |
+
}
|
| 112 |
+
segments.push_back(seg);
|
| 113 |
+
}
|
| 114 |
+
std::string out = "/";
|
| 115 |
+
for (size_t i = 0; i < segments.size(); ++i) {
|
| 116 |
+
if (i) out += "/";
|
| 117 |
+
out += segments[i];
|
| 118 |
+
}
|
| 119 |
+
return out;
|
| 120 |
}
|
| 121 |
|
| 122 |
std::string truncate_body(const std::string &body, size_t max_len) {
|
|
|
|
| 135 |
return request_id;
|
| 136 |
}
|
| 137 |
|
| 138 |
+
std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::shared_ptr<RequestContext> &ctx) {
|
| 139 |
asio::io_context ioc;
|
| 140 |
asio::ip::tcp::resolver resolver(ioc);
|
| 141 |
beast::tcp_stream stream(ioc);
|
| 142 |
auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
|
| 143 |
stream.connect(results);
|
| 144 |
+
// Generation can legitimately take minutes; the client-side request
|
| 145 |
+
// timeout cancels (and thereby unblocks) this read long before this
|
| 146 |
+
// safety net fires.
|
| 147 |
+
set_socket_timeouts(stream.socket().native_handle(), 3600, 60);
|
| 148 |
+
|
| 149 |
+
// Register the abort hook: a cancel from another thread shuts the socket
|
| 150 |
+
// down, which makes the blocking read below fail. The guard disarms it
|
| 151 |
+
// before the socket is destroyed so the hook can never touch a dead fd.
|
| 152 |
+
const int fd = stream.socket().native_handle();
|
| 153 |
+
struct AbortGuard {
|
| 154 |
+
const std::shared_ptr<RequestContext> &ctx;
|
| 155 |
+
~AbortGuard() { ctx->disarm_abort(); }
|
| 156 |
+
} abort_guard{ctx};
|
| 157 |
+
if (!ctx->arm_abort([fd]() { ::shutdown(fd, SHUT_RDWR); })) {
|
| 158 |
+
throw std::runtime_error("request cancelled before upstream call");
|
| 159 |
+
}
|
| 160 |
|
| 161 |
http::request<http::string_body> req{http::verb::post, "/v1/chat/completions", 11};
|
| 162 |
req.set(http::field::host, "127.0.0.1");
|
| 163 |
req.set(http::field::content_type, "application/json");
|
| 164 |
req.set(http::field::user_agent, "llm-manager");
|
| 165 |
+
req.body() = ctx->request_body;
|
| 166 |
req.prepare_payload();
|
| 167 |
http::write(stream, req);
|
| 168 |
|
|
|
|
| 180 |
beast::tcp_stream stream(ioc);
|
| 181 |
auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
|
| 182 |
stream.connect(results);
|
| 183 |
+
set_socket_timeouts(stream.socket().native_handle(), 30, 30);
|
| 184 |
|
| 185 |
http::request<http::string_body> req{http::verb::get, target, 11};
|
| 186 |
req.set(http::field::host, "127.0.0.1");
|
cpp/http_helpers.h
CHANGED
|
@@ -7,8 +7,20 @@
|
|
| 7 |
#include "llm_manager_types.h"
|
| 8 |
|
| 9 |
std::string now_utc_iso();
|
|
|
|
|
|
|
| 10 |
void log_line(const std::string &line);
|
| 11 |
std::string truncate_body(const std::string &body, size_t max_len = 2000);
|
| 12 |
std::optional<std::string> extract_cancel_request_id(const std::string &path);
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
ProxiedGetResult forward_get_to_worker(const WorkerInfo &worker, const std::string &target);
|
|
|
|
|
|
|
|
|
| 7 |
#include "llm_manager_types.h"
|
| 8 |
|
| 9 |
std::string now_utc_iso();
|
| 10 |
+
// Writes one log line; control characters in `line` are escaped so a client
|
| 11 |
+
// cannot forge log entries.
|
| 12 |
void log_line(const std::string &line);
|
| 13 |
std::string truncate_body(const std::string &body, size_t max_len = 2000);
|
| 14 |
std::optional<std::string> extract_cancel_request_id(const std::string &path);
|
| 15 |
+
// Percent-decodes and canonicalises a URL path (collapses "//", resolves "."
|
| 16 |
+
// and ".."). Returns nullopt for paths that escape the root or contain
|
| 17 |
+
// control characters, so callers can match against an exact allowlist.
|
| 18 |
+
std::optional<std::string> normalize_path(const std::string &raw_path);
|
| 19 |
+
// Generates a 128-bit random request id as lowercase hex.
|
| 20 |
+
std::string generate_request_id();
|
| 21 |
+
// Forwards the request to the worker. While the upstream connection is open
|
| 22 |
+
// it is registered on `ctx` so a concurrent cancel can shut it down.
|
| 23 |
+
std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::shared_ptr<RequestContext> &ctx);
|
| 24 |
ProxiedGetResult forward_get_to_worker(const WorkerInfo &worker, const std::string &target);
|
| 25 |
+
// Applies SO_RCVTIMEO/SO_SNDTIMEO to a native socket.
|
| 26 |
+
void set_socket_timeouts(int fd, int recv_sec, int send_sec);
|
cpp/llm_manager.cpp
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
#include <boost/asio.hpp>
|
| 2 |
|
|
|
|
| 3 |
#include <chrono>
|
| 4 |
#include <functional>
|
| 5 |
#include <string>
|
|
@@ -22,15 +23,30 @@ int main() {
|
|
| 22 |
ModelManager manager(config);
|
| 23 |
ApiKeyAuth auth(config);
|
| 24 |
RateLimiterStore rate_limiter(config.rate_limit);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
RequestRegistry registry;
|
| 26 |
MetricsRegistry metrics;
|
| 27 |
Scheduler scheduler(manager, registry, metrics, config.queue);
|
|
|
|
|
|
|
| 28 |
|
| 29 |
std::string init_error;
|
| 30 |
if (auth.enabled()) {
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
} else {
|
| 33 |
-
log_line("auth:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
}
|
| 35 |
|
| 36 |
log_line("startup: loading default model");
|
|
@@ -57,15 +73,28 @@ int main() {
|
|
| 57 |
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
| 58 |
continue;
|
| 59 |
}
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
std::
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
}
|
| 71 |
}
|
|
|
|
| 1 |
#include <boost/asio.hpp>
|
| 2 |
|
| 3 |
+
#include <atomic>
|
| 4 |
#include <chrono>
|
| 5 |
#include <functional>
|
| 6 |
#include <string>
|
|
|
|
| 23 |
ModelManager manager(config);
|
| 24 |
ApiKeyAuth auth(config);
|
| 25 |
RateLimiterStore rate_limiter(config.rate_limit);
|
| 26 |
+
RateLimitConfig ip_limit_config;
|
| 27 |
+
ip_limit_config.requests_per_minute = config.rate_limit.per_ip_requests_per_minute;
|
| 28 |
+
ip_limit_config.estimated_tokens_per_minute = 0;
|
| 29 |
+
RateLimiterStore ip_rate_limiter(ip_limit_config);
|
| 30 |
RequestRegistry registry;
|
| 31 |
MetricsRegistry metrics;
|
| 32 |
Scheduler scheduler(manager, registry, metrics, config.queue);
|
| 33 |
+
std::atomic<int> active_sessions{0};
|
| 34 |
+
ServerDeps deps{manager, config, auth, rate_limiter, ip_rate_limiter, registry, metrics, scheduler, active_sessions};
|
| 35 |
|
| 36 |
std::string init_error;
|
| 37 |
if (auth.enabled()) {
|
| 38 |
+
size_t usable = 0;
|
| 39 |
+
for (const auto &k : config.api_keys) if (k.enabled) ++usable;
|
| 40 |
+
log_line("auth: enabled api_keys=" + std::to_string(usable) + "/" + std::to_string(config.api_keys.size()) +
|
| 41 |
+
(config.auth.allow_anonymous ? " allow_anonymous=true(role=user)" : ""));
|
| 42 |
+
} else if (config.auth.allow_anonymous) {
|
| 43 |
+
log_line("auth: WARNING no usable api keys; anonymous access enabled with USER role");
|
| 44 |
} else {
|
| 45 |
+
log_line("auth: no usable api keys configured — all authenticated endpoints will return 401 "
|
| 46 |
+
"(set API_KEYS or [[api_keys]], or AUTH_ALLOW_ANONYMOUS=true)");
|
| 47 |
+
}
|
| 48 |
+
if (config.models.allowed.size() <= 1) {
|
| 49 |
+
log_line("models: allowlist contains only the default model (set ALLOWED_MODELS to permit others)");
|
| 50 |
}
|
| 51 |
|
| 52 |
log_line("startup: loading default model");
|
|
|
|
| 73 |
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
| 74 |
continue;
|
| 75 |
}
|
| 76 |
+
// Connection cap: each session pins a thread, so refuse fast once the
|
| 77 |
+
// configured maximum is reached instead of letting a flood exhaust us.
|
| 78 |
+
if (active_sessions.load() >= config.server.max_sessions) {
|
| 79 |
+
static const std::string kBusy =
|
| 80 |
+
"HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
|
| 81 |
+
"Retry-After: 2\r\nConnection: close\r\nContent-Length: 29\r\n\r\n"
|
| 82 |
+
"{\"error\":\"Too many sessions\"}";
|
| 83 |
+
boost::system::error_code ignored;
|
| 84 |
+
asio::write(socket, asio::buffer(kBusy), ignored);
|
| 85 |
+
socket.shutdown(asio::ip::tcp::socket::shutdown_both, ignored);
|
| 86 |
+
socket.close(ignored);
|
| 87 |
+
continue;
|
| 88 |
+
}
|
| 89 |
+
active_sessions.fetch_add(1);
|
| 90 |
+
try {
|
| 91 |
+
std::thread(&do_session, std::move(socket), std::ref(deps)).detach();
|
| 92 |
+
} catch (const std::exception &e) {
|
| 93 |
+
// Thread creation failed (resource exhaustion): the socket was
|
| 94 |
+
// moved into the thread argument and is gone; just release the slot.
|
| 95 |
+
active_sessions.fetch_sub(1);
|
| 96 |
+
log_line("thread_spawn_error=" + std::string(e.what()));
|
| 97 |
+
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
| 98 |
+
}
|
| 99 |
}
|
| 100 |
}
|
cpp/llm_manager_types.h
CHANGED
|
@@ -3,6 +3,7 @@
|
|
| 3 |
#include <atomic>
|
| 4 |
#include <chrono>
|
| 5 |
#include <condition_variable>
|
|
|
|
| 6 |
#include <memory>
|
| 7 |
#include <mutex>
|
| 8 |
#include <string>
|
|
@@ -17,6 +18,12 @@ using json = nlohmann::json;
|
|
| 17 |
struct ServerConfig {
|
| 18 |
std::string host = "0.0.0.0";
|
| 19 |
int port = 7860;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
};
|
| 21 |
|
| 22 |
struct WorkerConfig {
|
|
@@ -47,12 +54,20 @@ enum class Role {
|
|
| 47 |
struct AuthConfig {
|
| 48 |
std::string header = "Authorization";
|
| 49 |
std::string scheme = "Bearer";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
};
|
| 51 |
|
| 52 |
struct LimitsConfig {
|
| 53 |
int default_max_tokens = 256;
|
| 54 |
int max_tokens_per_request = 2048;
|
| 55 |
int request_timeout_sec = 30;
|
|
|
|
|
|
|
| 56 |
};
|
| 57 |
|
| 58 |
struct QueueConfig {
|
|
@@ -65,6 +80,22 @@ struct QueueConfig {
|
|
| 65 |
struct RateLimitConfig {
|
| 66 |
int requests_per_minute = 60;
|
| 67 |
int estimated_tokens_per_minute = 6000;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
};
|
| 69 |
|
| 70 |
struct SchedulerConfig {
|
|
@@ -77,7 +108,11 @@ struct StreamingConfig {
|
|
| 77 |
|
| 78 |
struct ApiKeyRecord {
|
| 79 |
std::string key_id;
|
|
|
|
|
|
|
| 80 |
std::string secret;
|
|
|
|
|
|
|
| 81 |
Role role = Role::USER;
|
| 82 |
bool enabled = true;
|
| 83 |
};
|
|
@@ -92,6 +127,8 @@ struct ManagerConfig {
|
|
| 92 |
RateLimitConfig rate_limit;
|
| 93 |
SchedulerConfig scheduler;
|
| 94 |
StreamingConfig streaming;
|
|
|
|
|
|
|
| 95 |
std::vector<ApiKeyRecord> api_keys;
|
| 96 |
};
|
| 97 |
|
|
@@ -116,6 +153,8 @@ enum class RequestState {
|
|
| 116 |
};
|
| 117 |
|
| 118 |
struct TokenEstimate {
|
|
|
|
|
|
|
| 119 |
int prompt_tokens = 0;
|
| 120 |
int requested_max_tokens = 0;
|
| 121 |
int estimated_total_tokens = 0;
|
|
@@ -156,6 +195,30 @@ struct RequestContext {
|
|
| 156 |
std::condition_variable cv;
|
| 157 |
bool completed = false;
|
| 158 |
RequestResult result;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
};
|
| 160 |
|
| 161 |
struct ProxiedGetResult {
|
|
|
|
| 3 |
#include <atomic>
|
| 4 |
#include <chrono>
|
| 5 |
#include <condition_variable>
|
| 6 |
+
#include <functional>
|
| 7 |
#include <memory>
|
| 8 |
#include <mutex>
|
| 9 |
#include <string>
|
|
|
|
| 18 |
struct ServerConfig {
|
| 19 |
std::string host = "0.0.0.0";
|
| 20 |
int port = 7860;
|
| 21 |
+
// Hard cap on concurrently open client connections (each one pins a
|
| 22 |
+
// thread). Excess connections get a fast 503.
|
| 23 |
+
int max_sessions = 256;
|
| 24 |
+
// Per-recv timeout for client sockets: bounds idle keep-alive connections
|
| 25 |
+
// and stalled senders.
|
| 26 |
+
int read_timeout_sec = 30;
|
| 27 |
};
|
| 28 |
|
| 29 |
struct WorkerConfig {
|
|
|
|
| 54 |
struct AuthConfig {
|
| 55 |
std::string header = "Authorization";
|
| 56 |
std::string scheme = "Bearer";
|
| 57 |
+
// Fail closed by default: with no API keys configured every request is
|
| 58 |
+
// rejected. Setting this lets unauthenticated callers in with USER role
|
| 59 |
+
// (never ADMIN).
|
| 60 |
+
bool allow_anonymous = false;
|
| 61 |
+
// Secrets shorter than this are refused at load time.
|
| 62 |
+
int min_secret_length = 32;
|
| 63 |
};
|
| 64 |
|
| 65 |
struct LimitsConfig {
|
| 66 |
int default_max_tokens = 256;
|
| 67 |
int max_tokens_per_request = 2048;
|
| 68 |
int request_timeout_sec = 30;
|
| 69 |
+
// Upper bound on any request body; larger bodies get 413 before parsing.
|
| 70 |
+
int max_request_body_bytes = 1 * 1024 * 1024;
|
| 71 |
};
|
| 72 |
|
| 73 |
struct QueueConfig {
|
|
|
|
| 80 |
struct RateLimitConfig {
|
| 81 |
int requests_per_minute = 60;
|
| 82 |
int estimated_tokens_per_minute = 6000;
|
| 83 |
+
// Pre-auth per-source limit applied to every request (0 = disabled). The
|
| 84 |
+
// source is the first X-Forwarded-For hop when trust_proxy_headers is
|
| 85 |
+
// set, otherwise the TCP peer address.
|
| 86 |
+
int per_ip_requests_per_minute = 0;
|
| 87 |
+
bool trust_proxy_headers = true;
|
| 88 |
+
};
|
| 89 |
+
|
| 90 |
+
struct LoggingConfig {
|
| 91 |
+
// Request bodies carry user prompts; keep them out of logs unless
|
| 92 |
+
// explicitly enabled for debugging.
|
| 93 |
+
bool log_request_bodies = false;
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
struct ModelsConfig {
|
| 97 |
+
// Models /switch-model may load. Empty means only the default model.
|
| 98 |
+
std::vector<std::string> allowed;
|
| 99 |
};
|
| 100 |
|
| 101 |
struct SchedulerConfig {
|
|
|
|
| 108 |
|
| 109 |
struct ApiKeyRecord {
|
| 110 |
std::string key_id;
|
| 111 |
+
// Plaintext secret as loaded from config (may be empty when only the
|
| 112 |
+
// hash is configured). Never logged.
|
| 113 |
std::string secret;
|
| 114 |
+
// Lowercase hex SHA-256 of the secret; this is what auth compares.
|
| 115 |
+
std::string secret_sha256;
|
| 116 |
Role role = Role::USER;
|
| 117 |
bool enabled = true;
|
| 118 |
};
|
|
|
|
| 127 |
RateLimitConfig rate_limit;
|
| 128 |
SchedulerConfig scheduler;
|
| 129 |
StreamingConfig streaming;
|
| 130 |
+
LoggingConfig logging;
|
| 131 |
+
ModelsConfig models;
|
| 132 |
std::vector<ApiKeyRecord> api_keys;
|
| 133 |
};
|
| 134 |
|
|
|
|
| 153 |
};
|
| 154 |
|
| 155 |
struct TokenEstimate {
|
| 156 |
+
// All values are validated to be non-negative and <= max_tokens_per_request
|
| 157 |
+
// before a request is admitted, so they always fit in int.
|
| 158 |
int prompt_tokens = 0;
|
| 159 |
int requested_max_tokens = 0;
|
| 160 |
int estimated_total_tokens = 0;
|
|
|
|
| 195 |
std::condition_variable cv;
|
| 196 |
bool completed = false;
|
| 197 |
RequestResult result;
|
| 198 |
+
|
| 199 |
+
// Soft cancellation: while the scheduler has an upstream connection open
|
| 200 |
+
// for this request it registers a callback that shuts that socket down.
|
| 201 |
+
// Cancelling then aborts only this request instead of restarting the
|
| 202 |
+
// worker (llama-server drops the task when the client disconnects).
|
| 203 |
+
std::mutex abort_mu;
|
| 204 |
+
std::function<void()> abort_upstream;
|
| 205 |
+
|
| 206 |
+
// Returns false if the request was already cancelled, in which case the
|
| 207 |
+
// caller must not proceed with the upstream call.
|
| 208 |
+
bool arm_abort(std::function<void()> fn) {
|
| 209 |
+
std::lock_guard<std::mutex> lock(abort_mu);
|
| 210 |
+
if (cancelled.load()) return false;
|
| 211 |
+
abort_upstream = std::move(fn);
|
| 212 |
+
return true;
|
| 213 |
+
}
|
| 214 |
+
void disarm_abort() {
|
| 215 |
+
std::lock_guard<std::mutex> lock(abort_mu);
|
| 216 |
+
abort_upstream = nullptr;
|
| 217 |
+
}
|
| 218 |
+
void trigger_abort() {
|
| 219 |
+
std::lock_guard<std::mutex> lock(abort_mu);
|
| 220 |
+
if (abort_upstream) abort_upstream();
|
| 221 |
+
}
|
| 222 |
};
|
| 223 |
|
| 224 |
struct ProxiedGetResult {
|
cpp/model_manager.cpp
CHANGED
|
@@ -172,16 +172,14 @@ json ModelManager::models_view() {
|
|
| 172 |
json out;
|
| 173 |
out["status"] = (active_ && is_alive(active_->pid)) ? "ready" : "no_active_model";
|
| 174 |
out["switch_in_progress"] = switch_in_progress_;
|
|
|
|
|
|
|
| 175 |
if (active_ && is_alive(active_->pid)) {
|
| 176 |
out["current_model"] = active_->model;
|
| 177 |
out["last_loaded"] = active_->last_loaded;
|
| 178 |
-
out["active_pid"] = active_->pid;
|
| 179 |
-
out["active_port"] = active_->port;
|
| 180 |
} else {
|
| 181 |
out["current_model"] = nullptr;
|
| 182 |
out["last_loaded"] = nullptr;
|
| 183 |
-
out["active_pid"] = nullptr;
|
| 184 |
-
out["active_port"] = nullptr;
|
| 185 |
}
|
| 186 |
return out;
|
| 187 |
}
|
|
@@ -264,6 +262,9 @@ std::pair<int, std::string> ModelManager::http_get(int port, const std::string &
|
|
| 264 |
beast::tcp_stream stream(ioc);
|
| 265 |
auto const results = resolver.resolve(worker_host_, std::to_string(port));
|
| 266 |
stream.connect(results);
|
|
|
|
|
|
|
|
|
|
| 267 |
|
| 268 |
http::request<http::string_body> req{http::verb::get, target, 11};
|
| 269 |
req.set(http::field::host, worker_host_);
|
|
|
|
| 172 |
json out;
|
| 173 |
out["status"] = (active_ && is_alive(active_->pid)) ? "ready" : "no_active_model";
|
| 174 |
out["switch_in_progress"] = switch_in_progress_;
|
| 175 |
+
// Process id and port are internal details; keep them out of the
|
| 176 |
+
// unauthenticated status view.
|
| 177 |
if (active_ && is_alive(active_->pid)) {
|
| 178 |
out["current_model"] = active_->model;
|
| 179 |
out["last_loaded"] = active_->last_loaded;
|
|
|
|
|
|
|
| 180 |
} else {
|
| 181 |
out["current_model"] = nullptr;
|
| 182 |
out["last_loaded"] = nullptr;
|
|
|
|
|
|
|
| 183 |
}
|
| 184 |
return out;
|
| 185 |
}
|
|
|
|
| 262 |
beast::tcp_stream stream(ioc);
|
| 263 |
auto const results = resolver.resolve(worker_host_, std::to_string(port));
|
| 264 |
stream.connect(results);
|
| 265 |
+
// Readiness probe: a worker that accepts but never answers must not stall
|
| 266 |
+
// the switch forever.
|
| 267 |
+
set_socket_timeouts(stream.socket().native_handle(), 5, 5);
|
| 268 |
|
| 269 |
http::request<http::string_body> req{http::verb::get, target, 11};
|
| 270 |
req.set(http::field::host, worker_host_);
|
cpp/request_parsing.cpp
CHANGED
|
@@ -1,11 +1,12 @@
|
|
| 1 |
#include "request_parsing.h"
|
| 2 |
|
| 3 |
#include <algorithm>
|
|
|
|
| 4 |
#include <sstream>
|
| 5 |
|
| 6 |
-
static
|
| 7 |
if (text.empty()) return 0;
|
| 8 |
-
return std::max(1, static_cast<
|
| 9 |
}
|
| 10 |
|
| 11 |
static std::string flatten_json_content(const json &content) {
|
|
@@ -35,6 +36,7 @@ static std::string flatten_json_content(const json &content) {
|
|
| 35 |
std::optional<TokenEstimate> estimate_chat_tokens(
|
| 36 |
const json &payload,
|
| 37 |
const LimitsConfig &limits,
|
|
|
|
| 38 |
std::string &error) {
|
| 39 |
if (!payload.is_object()) {
|
| 40 |
error = "Expected JSON object";
|
|
@@ -45,38 +47,60 @@ std::optional<TokenEstimate> estimate_chat_tokens(
|
|
| 45 |
return std::nullopt;
|
| 46 |
}
|
| 47 |
|
| 48 |
-
|
| 49 |
-
|
|
|
|
|
|
|
| 50 |
if (payload.contains("max_tokens")) {
|
| 51 |
if (!payload["max_tokens"].is_number_integer()) {
|
| 52 |
error = "Expected integer 'max_tokens'";
|
| 53 |
return std::nullopt;
|
| 54 |
}
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
}
|
| 57 |
|
| 58 |
-
if (
|
| 59 |
// -1 (and other non-positive values) is a sentinel meaning "unlimited" — use the default
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
}
|
| 62 |
|
|
|
|
| 63 |
for (const auto &message : payload["messages"]) {
|
| 64 |
if (!message.is_object()) continue;
|
| 65 |
if (message.contains("role") && message["role"].is_string()) {
|
| 66 |
-
|
| 67 |
}
|
| 68 |
if (message.contains("content")) {
|
| 69 |
-
|
| 70 |
}
|
| 71 |
-
|
|
|
|
| 72 |
}
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
-
|
| 75 |
-
if (
|
| 76 |
error = "Estimated request tokens exceed configured limit";
|
| 77 |
return std::nullopt;
|
| 78 |
}
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
error.clear();
|
| 81 |
return estimate;
|
| 82 |
}
|
|
|
|
| 1 |
#include "request_parsing.h"
|
| 2 |
|
| 3 |
#include <algorithm>
|
| 4 |
+
#include <cstdint>
|
| 5 |
#include <sstream>
|
| 6 |
|
| 7 |
+
static int64_t estimate_text_tokens_rough(const std::string &text) {
|
| 8 |
if (text.empty()) return 0;
|
| 9 |
+
return std::max<int64_t>(1, static_cast<int64_t>((text.size() + 3) / 4));
|
| 10 |
}
|
| 11 |
|
| 12 |
static std::string flatten_json_content(const json &content) {
|
|
|
|
| 36 |
std::optional<TokenEstimate> estimate_chat_tokens(
|
| 37 |
const json &payload,
|
| 38 |
const LimitsConfig &limits,
|
| 39 |
+
size_t body_bytes,
|
| 40 |
std::string &error) {
|
| 41 |
if (!payload.is_object()) {
|
| 42 |
error = "Expected JSON object";
|
|
|
|
| 47 |
return std::nullopt;
|
| 48 |
}
|
| 49 |
|
| 50 |
+
// All arithmetic in 64 bits: an int-sized max_tokens plus a prompt used
|
| 51 |
+
// to wrap negative and sail past every budget check below.
|
| 52 |
+
const int64_t max_per_request = limits.max_tokens_per_request;
|
| 53 |
+
int64_t requested_max_tokens = limits.default_max_tokens;
|
| 54 |
if (payload.contains("max_tokens")) {
|
| 55 |
if (!payload["max_tokens"].is_number_integer()) {
|
| 56 |
error = "Expected integer 'max_tokens'";
|
| 57 |
return std::nullopt;
|
| 58 |
}
|
| 59 |
+
// Out-of-range integers (beyond int64) come back saturated by nlohmann
|
| 60 |
+
// as unsigned; treat anything that does not fit as over the limit.
|
| 61 |
+
if (payload["max_tokens"].is_number_unsigned() &&
|
| 62 |
+
payload["max_tokens"].get<uint64_t>() > static_cast<uint64_t>(INT64_MAX)) {
|
| 63 |
+
error = "'max_tokens' exceeds configured limit";
|
| 64 |
+
return std::nullopt;
|
| 65 |
+
}
|
| 66 |
+
requested_max_tokens = payload["max_tokens"].get<int64_t>();
|
| 67 |
}
|
| 68 |
|
| 69 |
+
if (requested_max_tokens <= 0) {
|
| 70 |
// -1 (and other non-positive values) is a sentinel meaning "unlimited" — use the default
|
| 71 |
+
requested_max_tokens = limits.default_max_tokens;
|
| 72 |
+
}
|
| 73 |
+
if (requested_max_tokens > max_per_request) {
|
| 74 |
+
error = "'max_tokens' exceeds configured limit";
|
| 75 |
+
return std::nullopt;
|
| 76 |
}
|
| 77 |
|
| 78 |
+
int64_t prompt_tokens = 0;
|
| 79 |
for (const auto &message : payload["messages"]) {
|
| 80 |
if (!message.is_object()) continue;
|
| 81 |
if (message.contains("role") && message["role"].is_string()) {
|
| 82 |
+
prompt_tokens += estimate_text_tokens_rough(message["role"].get<std::string>());
|
| 83 |
}
|
| 84 |
if (message.contains("content")) {
|
| 85 |
+
prompt_tokens += estimate_text_tokens_rough(flatten_json_content(message["content"]));
|
| 86 |
}
|
| 87 |
+
prompt_tokens += 4;
|
| 88 |
+
if (prompt_tokens > max_per_request) break; // already over; stop walking
|
| 89 |
}
|
| 90 |
+
// Floor the estimate on the raw body size so payload that the text walker
|
| 91 |
+
// does not see (tools, image parts, unknown keys) is still budgeted.
|
| 92 |
+
prompt_tokens = std::max(prompt_tokens, static_cast<int64_t>(body_bytes / 4));
|
| 93 |
|
| 94 |
+
const int64_t total = prompt_tokens + requested_max_tokens;
|
| 95 |
+
if (prompt_tokens > max_per_request || total > max_per_request) {
|
| 96 |
error = "Estimated request tokens exceed configured limit";
|
| 97 |
return std::nullopt;
|
| 98 |
}
|
| 99 |
|
| 100 |
+
TokenEstimate estimate;
|
| 101 |
+
estimate.prompt_tokens = static_cast<int>(prompt_tokens);
|
| 102 |
+
estimate.requested_max_tokens = static_cast<int>(requested_max_tokens);
|
| 103 |
+
estimate.estimated_total_tokens = static_cast<int>(total);
|
| 104 |
error.clear();
|
| 105 |
return estimate;
|
| 106 |
}
|
cpp/request_parsing.h
CHANGED
|
@@ -5,9 +5,13 @@
|
|
| 5 |
|
| 6 |
#include "llm_manager_types.h"
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
std::optional<TokenEstimate> estimate_chat_tokens(
|
| 9 |
const json &payload,
|
| 10 |
const LimitsConfig &limits,
|
|
|
|
| 11 |
std::string &error);
|
| 12 |
|
| 13 |
bool request_stream_enabled(const json &payload);
|
|
|
|
| 5 |
|
| 6 |
#include "llm_manager_types.h"
|
| 7 |
|
| 8 |
+
// `body_bytes` is the size of the raw request body: it is used as a floor for
|
| 9 |
+
// the prompt estimate so fields the text walker does not see (tools, image
|
| 10 |
+
// parts, unknown keys) still count against the budget.
|
| 11 |
std::optional<TokenEstimate> estimate_chat_tokens(
|
| 12 |
const json &payload,
|
| 13 |
const LimitsConfig &limits,
|
| 14 |
+
size_t body_bytes,
|
| 15 |
std::string &error);
|
| 16 |
|
| 17 |
bool request_stream_enabled(const json &payload);
|
cpp/runtime_components.cpp
CHANGED
|
@@ -3,6 +3,7 @@
|
|
| 3 |
#include "config.h"
|
| 4 |
#include "http_helpers.h"
|
| 5 |
#include "model_manager.h"
|
|
|
|
| 6 |
|
| 7 |
#include <algorithm>
|
| 8 |
#include <cctype>
|
|
@@ -124,6 +125,10 @@ std::shared_ptr<RequestContext> RequestRegistry::cancel_request(
|
|
| 124 |
if (previous_state) *previous_state = expected;
|
| 125 |
if (expected == RequestState::RUNNING) {
|
| 126 |
ctx->state.store(RequestState::CANCELLED);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
}
|
| 128 |
}
|
| 129 |
return ctx;
|
|
@@ -145,6 +150,7 @@ std::vector<std::shared_ptr<RequestContext>> RequestRegistry::cancel_all() {
|
|
| 145 |
ctx->cv.notify_all();
|
| 146 |
} else if (expected == RequestState::RUNNING) {
|
| 147 |
ctx->state.store(RequestState::CANCELLED);
|
|
|
|
| 148 |
}
|
| 149 |
out.push_back(ctx);
|
| 150 |
}
|
|
@@ -358,7 +364,7 @@ void Scheduler::worker_loop() {
|
|
| 358 |
}
|
| 359 |
|
| 360 |
try {
|
| 361 |
-
auto [status, body] = forward_chat(*worker, ctx
|
| 362 |
if (ctx->cancelled.load()) {
|
| 363 |
registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
|
| 364 |
continue;
|
|
@@ -368,39 +374,66 @@ void Scheduler::worker_loop() {
|
|
| 368 |
}
|
| 369 |
registry_.complete(ctx, RequestState::DONE, {status, body});
|
| 370 |
} catch (const std::exception &e) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
log_line("request_id=" + ctx->request_id + " scheduler_exception=" + std::string(e.what()));
|
| 372 |
-
registry_.complete(ctx, RequestState::FAILED, {
|
| 373 |
} catch (...) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
log_line("request_id=" + ctx->request_id + " scheduler_exception=unknown");
|
| 375 |
-
registry_.complete(ctx, RequestState::FAILED, {
|
| 376 |
}
|
| 377 |
}
|
| 378 |
}
|
| 379 |
|
| 380 |
ApiKeyAuth::ApiKeyAuth(const ManagerConfig &config)
|
| 381 |
-
: header_name_(config.auth.header),
|
|
|
|
|
|
|
| 382 |
for (const auto &record : config.api_keys) {
|
| 383 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 384 |
}
|
| 385 |
}
|
| 386 |
|
| 387 |
bool ApiKeyAuth::enabled() const {
|
| 388 |
-
return
|
| 389 |
}
|
| 390 |
|
| 391 |
std::optional<ApiKeyRecord> ApiKeyAuth::authenticate(
|
| 392 |
const http::request<http::string_body> &req,
|
| 393 |
std::string &error) const {
|
| 394 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 395 |
error.clear();
|
| 396 |
-
return ApiKeyRecord{"anonymous", "", Role::
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
}
|
| 398 |
|
| 399 |
const auto token = extract_bearer_token(req, error);
|
| 400 |
if (!token) return std::nullopt;
|
| 401 |
|
| 402 |
-
|
| 403 |
-
|
|
|
|
|
|
|
|
|
|
| 404 |
error = "Invalid API key";
|
| 405 |
return std::nullopt;
|
| 406 |
}
|
|
|
|
| 3 |
#include "config.h"
|
| 4 |
#include "http_helpers.h"
|
| 5 |
#include "model_manager.h"
|
| 6 |
+
#include "sha256.h"
|
| 7 |
|
| 8 |
#include <algorithm>
|
| 9 |
#include <cctype>
|
|
|
|
| 125 |
if (previous_state) *previous_state = expected;
|
| 126 |
if (expected == RequestState::RUNNING) {
|
| 127 |
ctx->state.store(RequestState::CANCELLED);
|
| 128 |
+
// Soft cancel: drop the upstream connection for this request only.
|
| 129 |
+
// llama-server abandons the task when its client disconnects, so
|
| 130 |
+
// no worker restart is needed and other users are unaffected.
|
| 131 |
+
ctx->trigger_abort();
|
| 132 |
}
|
| 133 |
}
|
| 134 |
return ctx;
|
|
|
|
| 150 |
ctx->cv.notify_all();
|
| 151 |
} else if (expected == RequestState::RUNNING) {
|
| 152 |
ctx->state.store(RequestState::CANCELLED);
|
| 153 |
+
ctx->trigger_abort();
|
| 154 |
}
|
| 155 |
out.push_back(ctx);
|
| 156 |
}
|
|
|
|
| 364 |
}
|
| 365 |
|
| 366 |
try {
|
| 367 |
+
auto [status, body] = forward_chat(*worker, ctx);
|
| 368 |
if (ctx->cancelled.load()) {
|
| 369 |
registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
|
| 370 |
continue;
|
|
|
|
| 374 |
}
|
| 375 |
registry_.complete(ctx, RequestState::DONE, {status, body});
|
| 376 |
} catch (const std::exception &e) {
|
| 377 |
+
if (ctx->cancelled.load()) {
|
| 378 |
+
// Expected: the cancel path shut the upstream socket under us.
|
| 379 |
+
registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
|
| 380 |
+
continue;
|
| 381 |
+
}
|
| 382 |
+
// Upstream details stay in the log; clients get a generic error.
|
| 383 |
log_line("request_id=" + ctx->request_id + " scheduler_exception=" + std::string(e.what()));
|
| 384 |
+
registry_.complete(ctx, RequestState::FAILED, {502, R"({"error":"Upstream inference error"})"});
|
| 385 |
} catch (...) {
|
| 386 |
+
if (ctx->cancelled.load()) {
|
| 387 |
+
registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
|
| 388 |
+
continue;
|
| 389 |
+
}
|
| 390 |
log_line("request_id=" + ctx->request_id + " scheduler_exception=unknown");
|
| 391 |
+
registry_.complete(ctx, RequestState::FAILED, {502, R"({"error":"Upstream inference error"})"});
|
| 392 |
}
|
| 393 |
}
|
| 394 |
}
|
| 395 |
|
| 396 |
ApiKeyAuth::ApiKeyAuth(const ManagerConfig &config)
|
| 397 |
+
: header_name_(config.auth.header),
|
| 398 |
+
scheme_(config.auth.scheme),
|
| 399 |
+
allow_anonymous_(config.auth.allow_anonymous) {
|
| 400 |
for (const auto &record : config.api_keys) {
|
| 401 |
+
// Disabled records are kept so a presented-but-disabled key reports
|
| 402 |
+
// "disabled" instead of "invalid"; only enabled ones count as usable.
|
| 403 |
+
if (record.secret_sha256.empty()) continue;
|
| 404 |
+
records_by_hash_.emplace(record.secret_sha256, record);
|
| 405 |
+
if (record.enabled) ++usable_keys_;
|
| 406 |
}
|
| 407 |
}
|
| 408 |
|
| 409 |
bool ApiKeyAuth::enabled() const {
|
| 410 |
+
return usable_keys_ > 0;
|
| 411 |
}
|
| 412 |
|
| 413 |
std::optional<ApiKeyRecord> ApiKeyAuth::authenticate(
|
| 414 |
const http::request<http::string_body> &req,
|
| 415 |
std::string &error) const {
|
| 416 |
+
const bool has_header = req.find(header_name_) != req.end();
|
| 417 |
+
|
| 418 |
+
// Fail closed: with no usable keys and no explicit anonymous opt-in,
|
| 419 |
+
// nothing is let through. Anonymous access is never more than USER.
|
| 420 |
+
if (!has_header && allow_anonymous_) {
|
| 421 |
error.clear();
|
| 422 |
+
return ApiKeyRecord{"anonymous", "", "", Role::USER, true};
|
| 423 |
+
}
|
| 424 |
+
if (!enabled() && !has_header) {
|
| 425 |
+
error = "Authentication not configured";
|
| 426 |
+
return std::nullopt;
|
| 427 |
}
|
| 428 |
|
| 429 |
const auto token = extract_bearer_token(req, error);
|
| 430 |
if (!token) return std::nullopt;
|
| 431 |
|
| 432 |
+
// Hash first so lookup time does not depend on the secret's bytes, then
|
| 433 |
+
// confirm with a constant-time compare of the digests.
|
| 434 |
+
const std::string digest = sha256_hex(*token);
|
| 435 |
+
const auto it = records_by_hash_.find(digest);
|
| 436 |
+
if (it == records_by_hash_.end() || !constant_time_equals(it->first, digest)) {
|
| 437 |
error = "Invalid API key";
|
| 438 |
return std::nullopt;
|
| 439 |
}
|
cpp/runtime_components.h
CHANGED
|
@@ -157,7 +157,10 @@ public:
|
|
| 157 |
private:
|
| 158 |
std::string header_name_;
|
| 159 |
std::string scheme_;
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
std::optional<std::string> extract_bearer_token(
|
| 163 |
const http::request<http::string_body> &req,
|
|
|
|
| 157 |
private:
|
| 158 |
std::string header_name_;
|
| 159 |
std::string scheme_;
|
| 160 |
+
bool allow_anonymous_ = false;
|
| 161 |
+
size_t usable_keys_ = 0;
|
| 162 |
+
// Keyed by lowercase hex SHA-256 of the secret.
|
| 163 |
+
std::unordered_map<std::string, ApiKeyRecord> records_by_hash_;
|
| 164 |
|
| 165 |
std::optional<std::string> extract_bearer_token(
|
| 166 |
const http::request<http::string_body> &req,
|
cpp/server.cpp
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
|
| 3 |
#include <boost/beast/core.hpp>
|
| 4 |
|
|
|
|
| 5 |
#include "http_helpers.h"
|
| 6 |
#include "model_manager.h"
|
| 7 |
#include "request_parsing.h"
|
|
@@ -9,17 +10,14 @@
|
|
| 9 |
|
| 10 |
#include <algorithm>
|
| 11 |
#include <atomic>
|
|
|
|
| 12 |
#include <sstream>
|
|
|
|
| 13 |
#include <utility>
|
| 14 |
|
| 15 |
-
#include <sys/socket.h>
|
| 16 |
-
#include <sys/time.h>
|
| 17 |
-
|
| 18 |
namespace beast = boost::beast;
|
| 19 |
namespace http = beast::http;
|
| 20 |
|
| 21 |
-
static std::atomic<uint64_t> g_req_id{1};
|
| 22 |
-
|
| 23 |
static std::string build_sse_event(const json &payload) {
|
| 24 |
return "data: " + payload.dump() + "\n\n";
|
| 25 |
}
|
|
@@ -231,21 +229,60 @@ static std::string build_buffered_stream_response(const std::string &completion_
|
|
| 231 |
return oss.str();
|
| 232 |
}
|
| 233 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
http::response<http::string_body> handle_request(
|
| 235 |
-
|
| 236 |
-
const
|
| 237 |
-
const ApiKeyAuth &auth,
|
| 238 |
-
RateLimiterStore &rate_limiter,
|
| 239 |
-
RequestRegistry ®istry,
|
| 240 |
-
MetricsRegistry &metrics,
|
| 241 |
-
Scheduler &scheduler,
|
| 242 |
http::request<http::string_body> &&req) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
const auto start = std::chrono::steady_clock::now();
|
| 244 |
-
const
|
| 245 |
-
const std::string request_id = std::to_string(req_id_num);
|
| 246 |
const std::string target = req.target().to_string();
|
| 247 |
const std::string method = req.method_string().to_string();
|
| 248 |
-
const
|
|
|
|
|
|
|
| 249 |
auto authenticated = std::optional<ApiKeyRecord>{};
|
| 250 |
|
| 251 |
metrics.inc_requests_total();
|
|
@@ -255,8 +292,14 @@ http::response<http::string_body> handle_request(
|
|
| 255 |
~InflightGuard() { metrics.dec_requests_inflight(); }
|
| 256 |
} inflight_guard{metrics};
|
| 257 |
|
| 258 |
-
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
log_line("request_id=" + request_id + " body=" + truncate_body(req.body()));
|
| 261 |
}
|
| 262 |
|
|
@@ -304,7 +347,9 @@ http::response<http::string_body> handle_request(
|
|
| 304 |
std::string auth_error;
|
| 305 |
authenticated = auth.authenticate(req, auth_error);
|
| 306 |
if (!authenticated) {
|
| 307 |
-
|
|
|
|
|
|
|
| 308 |
}
|
| 309 |
if (minimum_role == Role::ADMIN && authenticated->role != Role::ADMIN) {
|
| 310 |
return json_response(http::status::forbidden, {{"error", "Admin role required"}});
|
|
@@ -316,6 +361,24 @@ http::response<http::string_body> handle_request(
|
|
| 316 |
};
|
| 317 |
|
| 318 |
try {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
if (path == "/health" && req.method() == http::verb::get) {
|
| 320 |
return json_response(http::status::ok, manager.models_view());
|
| 321 |
}
|
|
@@ -352,6 +415,15 @@ http::response<http::string_body> handle_request(
|
|
| 352 |
if (model.empty()) {
|
| 353 |
return json_response(http::status::bad_request, {{"error", "Expected string 'model' or 'model_name'"}});
|
| 354 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
|
| 356 |
std::string err;
|
| 357 |
bool ok = manager.switch_model(model, err);
|
|
@@ -389,30 +461,26 @@ http::response<http::string_body> handle_request(
|
|
| 389 |
if (auto cancel_id = extract_cancel_request_id(path)) {
|
| 390 |
if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
|
| 391 |
auto ctx = registry.find(*cancel_id);
|
| 392 |
-
|
|
|
|
|
|
|
| 393 |
return json_response(http::status::not_found, {{"error", "Unknown request id"}});
|
| 394 |
}
|
| 395 |
-
if (authenticated->role != Role::ADMIN && authenticated->key_id != ctx->api_key_id) {
|
| 396 |
-
return json_response(http::status::forbidden, {{"error", "Cannot cancel another API key request"}});
|
| 397 |
-
}
|
| 398 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 399 |
RequestState previous_state = RequestState::QUEUED;
|
| 400 |
registry.cancel_request(*cancel_id, &previous_state);
|
| 401 |
const bool did_cancel = previous_state == RequestState::QUEUED ||
|
| 402 |
previous_state == RequestState::RUNNING;
|
| 403 |
if (did_cancel) metrics.add_cancellations_total();
|
| 404 |
-
std::string restart_error;
|
| 405 |
-
bool restarted = true;
|
| 406 |
-
if (previous_state == RequestState::RUNNING) {
|
| 407 |
-
restarted = manager.restart_active(restart_error);
|
| 408 |
-
if (restarted) metrics.inc_worker_restarts_total();
|
| 409 |
-
}
|
| 410 |
|
| 411 |
json payload = {
|
| 412 |
{"cancelled_request_id", *cancel_id},
|
| 413 |
{"state", state_to_string(ctx->state.load())}
|
| 414 |
};
|
| 415 |
-
if (!restarted) payload["restart_error"] = restart_error;
|
| 416 |
return json_response(http::status::ok, payload);
|
| 417 |
}
|
| 418 |
}
|
|
@@ -438,7 +506,7 @@ http::response<http::string_body> handle_request(
|
|
| 438 |
}
|
| 439 |
|
| 440 |
std::string token_error;
|
| 441 |
-
auto estimate = estimate_chat_tokens(payload, config.limits, token_error);
|
| 442 |
if (!estimate) {
|
| 443 |
return json_response(http::status::bad_request, {{"error", token_error}});
|
| 444 |
}
|
|
@@ -500,20 +568,15 @@ http::response<http::string_body> handle_request(
|
|
| 500 |
}
|
| 501 |
if (!finished) {
|
| 502 |
lock.unlock();
|
|
|
|
|
|
|
| 503 |
RequestState timed_out_state = RequestState::QUEUED;
|
| 504 |
registry.cancel_request(request_id, &timed_out_state);
|
| 505 |
metrics.add_cancellations_total();
|
| 506 |
-
std::string restart_error;
|
| 507 |
-
bool restarted = true;
|
| 508 |
-
if (timed_out_state == RequestState::RUNNING) {
|
| 509 |
-
restarted = manager.restart_active(restart_error);
|
| 510 |
-
if (restarted) metrics.inc_worker_restarts_total();
|
| 511 |
-
}
|
| 512 |
json timeout_payload = {
|
| 513 |
{"error", "Request timed out"},
|
| 514 |
{"state", state_to_string(ctx->state.load())}
|
| 515 |
};
|
| 516 |
-
if (!restarted) timeout_payload["restart_error"] = restart_error;
|
| 517 |
return json_response(http::status::gateway_timeout, timeout_payload);
|
| 518 |
}
|
| 519 |
|
|
@@ -558,17 +621,28 @@ http::response<http::string_body> handle_request(
|
|
| 558 |
}
|
| 559 |
|
| 560 |
if (req.method() == http::verb::get) {
|
| 561 |
-
//
|
| 562 |
-
//
|
| 563 |
-
//
|
| 564 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 565 |
if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
|
| 566 |
}
|
| 567 |
auto worker = manager.active_worker();
|
| 568 |
if (!worker) {
|
| 569 |
return json_response(http::status::service_unavailable, {{"error", "No active model"}});
|
| 570 |
}
|
| 571 |
-
auto upstream = forward_get_to_worker(*worker,
|
| 572 |
http::response<http::string_body> res{
|
| 573 |
static_cast<http::status>(upstream.status), req.version()};
|
| 574 |
res.set(http::field::content_type, upstream.content_type);
|
|
@@ -595,38 +669,60 @@ http::response<http::string_body> handle_request(
|
|
| 595 |
|
| 596 |
return json_response(http::status::not_found, {{"error", "Not found"}});
|
| 597 |
} catch (const std::exception &e) {
|
|
|
|
|
|
|
| 598 |
log_line("request_id=" + request_id + " exception=" + std::string(e.what()));
|
| 599 |
-
return json_response(http::status::internal_server_error, {{"error",
|
| 600 |
} catch (...) {
|
| 601 |
log_line("request_id=" + request_id + " exception=unknown");
|
| 602 |
-
return json_response(http::status::internal_server_error, {{"error", "
|
| 603 |
}
|
| 604 |
}
|
| 605 |
|
| 606 |
-
void do_session(
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
MetricsRegistry &metrics,
|
| 614 |
-
Scheduler &scheduler) {
|
| 615 |
// Receive timeout: each connection pins a detached thread, so idle
|
| 616 |
// keep-alive connections and stalled senders must eventually release it.
|
|
|
|
|
|
|
|
|
|
| 617 |
{
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
reinterpret_cast<const char *>(&tv), sizeof(tv));
|
| 622 |
}
|
|
|
|
| 623 |
try {
|
| 624 |
beast::flat_buffer buffer;
|
| 625 |
for (;;) {
|
| 626 |
-
|
| 627 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 628 |
const bool keep_alive = req.keep_alive();
|
| 629 |
-
auto res = handle_request(
|
| 630 |
http::write(socket, res);
|
| 631 |
if (!keep_alive || res.need_eof()) break;
|
| 632 |
}
|
|
|
|
| 2 |
|
| 3 |
#include <boost/beast/core.hpp>
|
| 4 |
|
| 5 |
+
#include "config.h"
|
| 6 |
#include "http_helpers.h"
|
| 7 |
#include "model_manager.h"
|
| 8 |
#include "request_parsing.h"
|
|
|
|
| 10 |
|
| 11 |
#include <algorithm>
|
| 12 |
#include <atomic>
|
| 13 |
+
#include <regex>
|
| 14 |
#include <sstream>
|
| 15 |
+
#include <unordered_set>
|
| 16 |
#include <utility>
|
| 17 |
|
|
|
|
|
|
|
|
|
|
| 18 |
namespace beast = boost::beast;
|
| 19 |
namespace http = beast::http;
|
| 20 |
|
|
|
|
|
|
|
| 21 |
static std::string build_sse_event(const json &payload) {
|
| 22 |
return "data: " + payload.dump() + "\n\n";
|
| 23 |
}
|
|
|
|
| 229 |
return oss.str();
|
| 230 |
}
|
| 231 |
|
| 232 |
+
// Model identifiers go straight onto llama-server's command line as the -hf
|
| 233 |
+
// value, so only accept the owner/repo[:quant] shape and nothing that looks
|
| 234 |
+
// like a flag, a path, or contains control characters.
|
| 235 |
+
static bool is_valid_model_identifier(const std::string &model) {
|
| 236 |
+
if (model.empty() || model.size() > 256) return false;
|
| 237 |
+
static const std::regex pattern(
|
| 238 |
+
R"(^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*(:[A-Za-z0-9][A-Za-z0-9_.-]*)?$)");
|
| 239 |
+
if (!std::regex_match(model, pattern)) return false;
|
| 240 |
+
return model.find("..") == std::string::npos;
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
static bool is_allowed_model(const ModelsConfig &models, const std::string &model) {
|
| 244 |
+
return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
|
| 245 |
+
}
|
| 246 |
+
|
| 247 |
+
// Resolves the client address used for per-source rate limiting. The first
|
| 248 |
+
// X-Forwarded-For hop is used only when the deployment says the proxy in
|
| 249 |
+
// front of us is trusted (HF Spaces); otherwise the TCP peer.
|
| 250 |
+
static std::string client_address(
|
| 251 |
+
const http::request<http::string_body> &req,
|
| 252 |
+
const std::string &peer_address,
|
| 253 |
+
bool trust_proxy_headers) {
|
| 254 |
+
if (trust_proxy_headers) {
|
| 255 |
+
const auto it = req.find("X-Forwarded-For");
|
| 256 |
+
if (it != req.end()) {
|
| 257 |
+
std::string value = it->value().to_string();
|
| 258 |
+
const auto comma = value.find(',');
|
| 259 |
+
if (comma != std::string::npos) value = value.substr(0, comma);
|
| 260 |
+
value = trim_copy(value);
|
| 261 |
+
if (!value.empty() && value.size() <= 64) return value;
|
| 262 |
+
}
|
| 263 |
+
}
|
| 264 |
+
return peer_address;
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
http::response<http::string_body> handle_request(
|
| 268 |
+
ServerDeps &deps,
|
| 269 |
+
const std::string &peer_address,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
http::request<http::string_body> &&req) {
|
| 271 |
+
ModelManager &manager = deps.manager;
|
| 272 |
+
const ManagerConfig &config = deps.config;
|
| 273 |
+
const ApiKeyAuth &auth = deps.auth;
|
| 274 |
+
RateLimiterStore &rate_limiter = deps.rate_limiter;
|
| 275 |
+
RequestRegistry ®istry = deps.registry;
|
| 276 |
+
MetricsRegistry &metrics = deps.metrics;
|
| 277 |
+
Scheduler &scheduler = deps.scheduler;
|
| 278 |
+
|
| 279 |
const auto start = std::chrono::steady_clock::now();
|
| 280 |
+
const std::string request_id = generate_request_id();
|
|
|
|
| 281 |
const std::string target = req.target().to_string();
|
| 282 |
const std::string method = req.method_string().to_string();
|
| 283 |
+
const auto query_pos = target.find('?');
|
| 284 |
+
const std::string raw_path = target.substr(0, query_pos);
|
| 285 |
+
const std::string query = query_pos == std::string::npos ? "" : target.substr(query_pos);
|
| 286 |
auto authenticated = std::optional<ApiKeyRecord>{};
|
| 287 |
|
| 288 |
metrics.inc_requests_total();
|
|
|
|
| 292 |
~InflightGuard() { metrics.dec_requests_inflight(); }
|
| 293 |
} inflight_guard{metrics};
|
| 294 |
|
| 295 |
+
// Canonical path for routing. Anything that does not normalise (escapes
|
| 296 |
+
// the root, control chars, bad percent-encoding) is simply not found.
|
| 297 |
+
const auto normalized = normalize_path(raw_path);
|
| 298 |
+
const std::string path = normalized ? *normalized : std::string();
|
| 299 |
+
|
| 300 |
+
log_line("request_id=" + request_id + " method=" + method + " path=" + raw_path +
|
| 301 |
+
" body_bytes=" + std::to_string(req.body().size()));
|
| 302 |
+
if (config.logging.log_request_bodies && !req.body().empty()) {
|
| 303 |
log_line("request_id=" + request_id + " body=" + truncate_body(req.body()));
|
| 304 |
}
|
| 305 |
|
|
|
|
| 347 |
std::string auth_error;
|
| 348 |
authenticated = auth.authenticate(req, auth_error);
|
| 349 |
if (!authenticated) {
|
| 350 |
+
auto res = json_response(http::status::unauthorized, {{"error", auth_error}});
|
| 351 |
+
res.set(http::field::www_authenticate, config.auth.scheme);
|
| 352 |
+
return res;
|
| 353 |
}
|
| 354 |
if (minimum_role == Role::ADMIN && authenticated->role != Role::ADMIN) {
|
| 355 |
return json_response(http::status::forbidden, {{"error", "Admin role required"}});
|
|
|
|
| 361 |
};
|
| 362 |
|
| 363 |
try {
|
| 364 |
+
// Pre-auth per-source limit: protects the thread pool and the auth
|
| 365 |
+
// path itself from a single abusive client.
|
| 366 |
+
if (config.rate_limit.per_ip_requests_per_minute > 0) {
|
| 367 |
+
const std::string source = client_address(req, peer_address, config.rate_limit.trust_proxy_headers);
|
| 368 |
+
auto decision = deps.ip_rate_limiter.allow("ip:" + source, 0);
|
| 369 |
+
if (!decision.allowed) {
|
| 370 |
+
metrics.inc_rate_limited_total();
|
| 371 |
+
return json_response_with_retry_after(
|
| 372 |
+
http::status::too_many_requests,
|
| 373 |
+
{{"error", "Rate limit exceeded: requests from this address"}},
|
| 374 |
+
decision.retry_after_sec);
|
| 375 |
+
}
|
| 376 |
+
}
|
| 377 |
+
|
| 378 |
+
if (!normalized) {
|
| 379 |
+
return json_response(http::status::not_found, {{"error", "Not found"}});
|
| 380 |
+
}
|
| 381 |
+
|
| 382 |
if (path == "/health" && req.method() == http::verb::get) {
|
| 383 |
return json_response(http::status::ok, manager.models_view());
|
| 384 |
}
|
|
|
|
| 415 |
if (model.empty()) {
|
| 416 |
return json_response(http::status::bad_request, {{"error", "Expected string 'model' or 'model_name'"}});
|
| 417 |
}
|
| 418 |
+
if (!is_valid_model_identifier(model)) {
|
| 419 |
+
return json_response(http::status::bad_request,
|
| 420 |
+
{{"error", "Invalid model identifier; expected owner/repo[:quant]"}});
|
| 421 |
+
}
|
| 422 |
+
if (!is_allowed_model(config.models, model)) {
|
| 423 |
+
return json_response(http::status::bad_request,
|
| 424 |
+
{{"error", "Model is not in the configured allowlist"},
|
| 425 |
+
{"allowed_models", config.models.allowed}});
|
| 426 |
+
}
|
| 427 |
|
| 428 |
std::string err;
|
| 429 |
bool ok = manager.switch_model(model, err);
|
|
|
|
| 461 |
if (auto cancel_id = extract_cancel_request_id(path)) {
|
| 462 |
if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
|
| 463 |
auto ctx = registry.find(*cancel_id);
|
| 464 |
+
// Same answer for "unknown" and "not yours" so request ids of
|
| 465 |
+
// other keys cannot be probed.
|
| 466 |
+
if (!ctx || (authenticated->role != Role::ADMIN && authenticated->key_id != ctx->api_key_id)) {
|
| 467 |
return json_response(http::status::not_found, {{"error", "Unknown request id"}});
|
| 468 |
}
|
|
|
|
|
|
|
|
|
|
| 469 |
|
| 470 |
+
// Soft cancel only: a queued request is dropped, a running one
|
| 471 |
+
// has its upstream connection closed. The worker is never
|
| 472 |
+
// restarted from here (that is the admin-only /stop), so one
|
| 473 |
+
// key cannot take the service down for everyone else.
|
| 474 |
RequestState previous_state = RequestState::QUEUED;
|
| 475 |
registry.cancel_request(*cancel_id, &previous_state);
|
| 476 |
const bool did_cancel = previous_state == RequestState::QUEUED ||
|
| 477 |
previous_state == RequestState::RUNNING;
|
| 478 |
if (did_cancel) metrics.add_cancellations_total();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
|
| 480 |
json payload = {
|
| 481 |
{"cancelled_request_id", *cancel_id},
|
| 482 |
{"state", state_to_string(ctx->state.load())}
|
| 483 |
};
|
|
|
|
| 484 |
return json_response(http::status::ok, payload);
|
| 485 |
}
|
| 486 |
}
|
|
|
|
| 506 |
}
|
| 507 |
|
| 508 |
std::string token_error;
|
| 509 |
+
auto estimate = estimate_chat_tokens(payload, config.limits, req.body().size(), token_error);
|
| 510 |
if (!estimate) {
|
| 511 |
return json_response(http::status::bad_request, {{"error", token_error}});
|
| 512 |
}
|
|
|
|
| 568 |
}
|
| 569 |
if (!finished) {
|
| 570 |
lock.unlock();
|
| 571 |
+
// Soft cancel: closes this request's upstream connection; the
|
| 572 |
+
// worker keeps serving everyone else.
|
| 573 |
RequestState timed_out_state = RequestState::QUEUED;
|
| 574 |
registry.cancel_request(request_id, &timed_out_state);
|
| 575 |
metrics.add_cancellations_total();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
json timeout_payload = {
|
| 577 |
{"error", "Request timed out"},
|
| 578 |
{"state", state_to_string(ctx->state.load())}
|
| 579 |
};
|
|
|
|
| 580 |
return json_response(http::status::gateway_timeout, timeout_payload);
|
| 581 |
}
|
| 582 |
|
|
|
|
| 621 |
}
|
| 622 |
|
| 623 |
if (req.method() == http::verb::get) {
|
| 624 |
+
// Allowlist, not blocklist: only the paths the bundled web UI needs
|
| 625 |
+
// are proxied, matched on the canonical (decoded) path so encoded
|
| 626 |
+
// variants cannot slip past. /props exposes model and template
|
| 627 |
+
// settings, /slots other users' prompts, /metrics server internals
|
| 628 |
+
// — those need a key.
|
| 629 |
+
static const std::unordered_set<std::string> kPublicPaths = {
|
| 630 |
+
"/", "/index.html", "/index.html.gz", "/favicon.ico"};
|
| 631 |
+
static const std::unordered_set<std::string> kAuthenticatedPaths = {
|
| 632 |
+
"/props", "/v1/models", "/slots", "/metrics"};
|
| 633 |
+
const bool is_public = kPublicPaths.count(path) > 0;
|
| 634 |
+
const bool needs_auth = kAuthenticatedPaths.count(path) > 0;
|
| 635 |
+
if (!is_public && !needs_auth) {
|
| 636 |
+
return json_response(http::status::not_found, {{"error", "Not found"}});
|
| 637 |
+
}
|
| 638 |
+
if (needs_auth) {
|
| 639 |
if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
|
| 640 |
}
|
| 641 |
auto worker = manager.active_worker();
|
| 642 |
if (!worker) {
|
| 643 |
return json_response(http::status::service_unavailable, {{"error", "No active model"}});
|
| 644 |
}
|
| 645 |
+
auto upstream = forward_get_to_worker(*worker, path + query);
|
| 646 |
http::response<http::string_body> res{
|
| 647 |
static_cast<http::status>(upstream.status), req.version()};
|
| 648 |
res.set(http::field::content_type, upstream.content_type);
|
|
|
|
| 669 |
|
| 670 |
return json_response(http::status::not_found, {{"error", "Not found"}});
|
| 671 |
} catch (const std::exception &e) {
|
| 672 |
+
// Exception text can reveal internals (paths, library messages); log
|
| 673 |
+
// it, return a generic message.
|
| 674 |
log_line("request_id=" + request_id + " exception=" + std::string(e.what()));
|
| 675 |
+
return json_response(http::status::internal_server_error, {{"error", "Internal server error"}});
|
| 676 |
} catch (...) {
|
| 677 |
log_line("request_id=" + request_id + " exception=unknown");
|
| 678 |
+
return json_response(http::status::internal_server_error, {{"error", "Internal server error"}});
|
| 679 |
}
|
| 680 |
}
|
| 681 |
|
| 682 |
+
void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
|
| 683 |
+
struct SessionGuard {
|
| 684 |
+
std::atomic<int> &counter;
|
| 685 |
+
~SessionGuard() { counter.fetch_sub(1); }
|
| 686 |
+
} session_guard{deps.active_sessions};
|
| 687 |
+
|
| 688 |
+
const ManagerConfig &config = deps.config;
|
|
|
|
|
|
|
| 689 |
// Receive timeout: each connection pins a detached thread, so idle
|
| 690 |
// keep-alive connections and stalled senders must eventually release it.
|
| 691 |
+
set_socket_timeouts(socket.native_handle(), config.server.read_timeout_sec, 60);
|
| 692 |
+
|
| 693 |
+
std::string peer_address;
|
| 694 |
{
|
| 695 |
+
boost::system::error_code ec;
|
| 696 |
+
const auto endpoint = socket.remote_endpoint(ec);
|
| 697 |
+
if (!ec) peer_address = endpoint.address().to_string();
|
|
|
|
| 698 |
}
|
| 699 |
+
|
| 700 |
try {
|
| 701 |
beast::flat_buffer buffer;
|
| 702 |
for (;;) {
|
| 703 |
+
// Explicit limits instead of Beast defaults: oversized headers or
|
| 704 |
+
// bodies are rejected before they are buffered in full.
|
| 705 |
+
http::request_parser<http::string_body> parser;
|
| 706 |
+
parser.header_limit(16 * 1024);
|
| 707 |
+
parser.body_limit(static_cast<uint64_t>(config.limits.max_request_body_bytes));
|
| 708 |
+
beast::error_code read_ec;
|
| 709 |
+
http::read(socket, buffer, parser, read_ec);
|
| 710 |
+
if (read_ec == http::error::body_limit || read_ec == http::error::header_limit) {
|
| 711 |
+
http::response<http::string_body> res{http::status::payload_too_large, 11};
|
| 712 |
+
res.set(http::field::content_type, "application/json");
|
| 713 |
+
res.set(http::field::server, "llm-manager");
|
| 714 |
+
res.keep_alive(false);
|
| 715 |
+
res.body() = R"({"error":"Request too large"})";
|
| 716 |
+
res.prepare_payload();
|
| 717 |
+
beast::error_code write_ec;
|
| 718 |
+
http::write(socket, res, write_ec);
|
| 719 |
+
break;
|
| 720 |
+
}
|
| 721 |
+
if (read_ec) throw beast::system_error{read_ec};
|
| 722 |
+
|
| 723 |
+
http::request<http::string_body> req = parser.release();
|
| 724 |
const bool keep_alive = req.keep_alive();
|
| 725 |
+
auto res = handle_request(deps, peer_address, std::move(req));
|
| 726 |
http::write(socket, res);
|
| 727 |
if (!keep_alive || res.need_eof()) break;
|
| 728 |
}
|
cpp/server.h
CHANGED
|
@@ -3,6 +3,8 @@
|
|
| 3 |
#include <boost/asio.hpp>
|
| 4 |
#include <boost/beast/http.hpp>
|
| 5 |
|
|
|
|
|
|
|
| 6 |
#include "llm_manager_types.h"
|
| 7 |
|
| 8 |
class ModelManager;
|
|
@@ -14,22 +16,26 @@ class Scheduler;
|
|
| 14 |
|
| 15 |
namespace http = boost::beast::http;
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
http::response<http::string_body> handle_request(
|
| 18 |
-
|
| 19 |
-
const
|
| 20 |
-
const ApiKeyAuth &auth,
|
| 21 |
-
RateLimiterStore &rate_limiter,
|
| 22 |
-
RequestRegistry ®istry,
|
| 23 |
-
MetricsRegistry &metrics,
|
| 24 |
-
Scheduler &scheduler,
|
| 25 |
http::request<http::string_body> &&req);
|
| 26 |
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
const ManagerConfig &config,
|
| 31 |
-
const ApiKeyAuth &auth,
|
| 32 |
-
RateLimiterStore &rate_limiter,
|
| 33 |
-
RequestRegistry ®istry,
|
| 34 |
-
MetricsRegistry &metrics,
|
| 35 |
-
Scheduler &scheduler);
|
|
|
|
| 3 |
#include <boost/asio.hpp>
|
| 4 |
#include <boost/beast/http.hpp>
|
| 5 |
|
| 6 |
+
#include <atomic>
|
| 7 |
+
|
| 8 |
#include "llm_manager_types.h"
|
| 9 |
|
| 10 |
class ModelManager;
|
|
|
|
| 16 |
|
| 17 |
namespace http = boost::beast::http;
|
| 18 |
|
| 19 |
+
// Everything a session needs; owned by main() for the process lifetime.
|
| 20 |
+
struct ServerDeps {
|
| 21 |
+
ModelManager &manager;
|
| 22 |
+
const ManagerConfig &config;
|
| 23 |
+
const ApiKeyAuth &auth;
|
| 24 |
+
RateLimiterStore &rate_limiter;
|
| 25 |
+
// Pre-auth per-source limiter (keyed by client address).
|
| 26 |
+
RateLimiterStore &ip_rate_limiter;
|
| 27 |
+
RequestRegistry ®istry;
|
| 28 |
+
MetricsRegistry &metrics;
|
| 29 |
+
Scheduler &scheduler;
|
| 30 |
+
// Open sessions, for the connection cap.
|
| 31 |
+
std::atomic<int> &active_sessions;
|
| 32 |
+
};
|
| 33 |
+
|
| 34 |
http::response<http::string_body> handle_request(
|
| 35 |
+
ServerDeps &deps,
|
| 36 |
+
const std::string &peer_address,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
http::request<http::string_body> &&req);
|
| 38 |
|
| 39 |
+
// Takes ownership of one accepted connection; decrements
|
| 40 |
+
// deps.active_sessions on exit.
|
| 41 |
+
void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cpp/sha256.h
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
// Minimal, dependency-free SHA-256 used to store and compare API key secrets
|
| 4 |
+
// as digests. Not performance critical: one hash per authenticated request.
|
| 5 |
+
|
| 6 |
+
#include <array>
|
| 7 |
+
#include <cstdint>
|
| 8 |
+
#include <cstring>
|
| 9 |
+
#include <string>
|
| 10 |
+
|
| 11 |
+
namespace sha256_detail {
|
| 12 |
+
|
| 13 |
+
inline uint32_t rotr(uint32_t x, uint32_t n) { return (x >> n) | (x << (32 - n)); }
|
| 14 |
+
|
| 15 |
+
constexpr uint32_t K[64] = {
|
| 16 |
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
| 17 |
+
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
| 18 |
+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
| 19 |
+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
| 20 |
+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
| 21 |
+
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
| 22 |
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
| 23 |
+
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
|
| 24 |
+
|
| 25 |
+
inline void compress(std::array<uint32_t, 8> &h, const uint8_t block[64]) {
|
| 26 |
+
uint32_t w[64];
|
| 27 |
+
for (int i = 0; i < 16; ++i) {
|
| 28 |
+
w[i] = (uint32_t(block[i * 4]) << 24) | (uint32_t(block[i * 4 + 1]) << 16) |
|
| 29 |
+
(uint32_t(block[i * 4 + 2]) << 8) | uint32_t(block[i * 4 + 3]);
|
| 30 |
+
}
|
| 31 |
+
for (int i = 16; i < 64; ++i) {
|
| 32 |
+
const uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
|
| 33 |
+
const uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
|
| 34 |
+
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
|
| 35 |
+
}
|
| 36 |
+
uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7];
|
| 37 |
+
for (int i = 0; i < 64; ++i) {
|
| 38 |
+
const uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
| 39 |
+
const uint32_t ch = (e & f) ^ (~e & g);
|
| 40 |
+
const uint32_t t1 = hh + S1 + ch + K[i] + w[i];
|
| 41 |
+
const uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
| 42 |
+
const uint32_t maj = (a & b) ^ (a & c) ^ (b & c);
|
| 43 |
+
const uint32_t t2 = S0 + maj;
|
| 44 |
+
hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2;
|
| 45 |
+
}
|
| 46 |
+
h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += hh;
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
} // namespace sha256_detail
|
| 50 |
+
|
| 51 |
+
// Lowercase hex digest of `input`.
|
| 52 |
+
inline std::string sha256_hex(const std::string &input) {
|
| 53 |
+
std::array<uint32_t, 8> h = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
|
| 54 |
+
0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
|
| 55 |
+
const uint8_t *data = reinterpret_cast<const uint8_t *>(input.data());
|
| 56 |
+
size_t len = input.size();
|
| 57 |
+
while (len >= 64) {
|
| 58 |
+
sha256_detail::compress(h, data);
|
| 59 |
+
data += 64;
|
| 60 |
+
len -= 64;
|
| 61 |
+
}
|
| 62 |
+
uint8_t block[128] = {0};
|
| 63 |
+
std::memcpy(block, data, len);
|
| 64 |
+
block[len] = 0x80;
|
| 65 |
+
const size_t total = (len + 1 + 8 <= 64) ? 64 : 128;
|
| 66 |
+
const uint64_t bit_len = uint64_t(input.size()) * 8;
|
| 67 |
+
for (int i = 0; i < 8; ++i) block[total - 1 - i] = uint8_t(bit_len >> (8 * i));
|
| 68 |
+
sha256_detail::compress(h, block);
|
| 69 |
+
if (total == 128) sha256_detail::compress(h, block + 64);
|
| 70 |
+
|
| 71 |
+
static const char *digits = "0123456789abcdef";
|
| 72 |
+
std::string out;
|
| 73 |
+
out.reserve(64);
|
| 74 |
+
for (uint32_t v : h) {
|
| 75 |
+
for (int shift = 28; shift >= 0; shift -= 4) out.push_back(digits[(v >> shift) & 0xF]);
|
| 76 |
+
}
|
| 77 |
+
return out;
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
// Compares two strings without early exit on the first differing byte.
|
| 81 |
+
inline bool constant_time_equals(const std::string &a, const std::string &b) {
|
| 82 |
+
if (a.size() != b.size()) return false;
|
| 83 |
+
unsigned char diff = 0;
|
| 84 |
+
for (size_t i = 0; i < a.size(); ++i) {
|
| 85 |
+
diff |= static_cast<unsigned char>(a[i]) ^ static_cast<unsigned char>(b[i]);
|
| 86 |
+
}
|
| 87 |
+
return diff == 0;
|
| 88 |
+
}
|