Dmitry Beresnev commited on
Commit
6471a06
·
1 Parent(s): 27c1c3c

fix critical secutity issues

Browse files
.gitignore CHANGED
@@ -72,6 +72,9 @@ cover/
72
  logs/
73
  agi.log
74
 
 
 
 
75
  # Environment variables
76
  .env
77
  .env.local
 
72
  logs/
73
  agi.log
74
 
75
+ # Runtime config holds API key secrets — never commit it
76
+ config.toml
77
+
78
  # Environment variables
79
  .env
80
  .env.local
Dockerfile CHANGED
@@ -18,7 +18,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
18
  # Clone and build llama.cpp with SSL support for HuggingFace Hub
19
  WORKDIR /build
20
  ARG CACHEBUST=6
21
- ARG LLAMA_CPP_REF=master
 
 
22
  #ARG BUILD_PROFILE=fast_build
23
  ARG BUILD_PROFILE=fast_runtime
24
  ARG BUILD_JOBS=1
@@ -91,32 +93,22 @@ COPY --from=builder /build/llm-manager /usr/local/bin/llm-manager
91
  # Update library cache
92
  RUN ldconfig
93
 
94
- # Install Python and FastAPI dependencies
95
- RUN apt-get update && apt-get install -y --no-install-recommends \
96
- python3 \
97
- python3-pip \
98
- && rm -rf /var/lib/apt/lists/*
99
-
100
- # Install Python packages
101
- RUN pip3 install --no-cache-dir fastapi uvicorn aiohttp pydantic duckduckgo-search beautifulsoup4 lxml --break-system-packages
102
 
103
  # Create non-root user
104
  RUN useradd -m -u 1000 user && \
105
  mkdir -p /home/user/.cache/llama.cpp && \
106
  chown -R user:user /home/user
107
 
108
- # Copy application code
109
- COPY --chown=user:user python/ /home/user/python/
110
-
111
  USER user
112
  WORKDIR /home/user
113
 
114
  # Set environment variables
115
  ENV HOME=/home/user \
116
  LLAMA_CACHE=/home/user/.cache/llama.cpp \
117
- PATH=/home/user/.local/bin:$PATH \
118
- PYTHONPATH=/home/user/python \
119
- PYTHONUNBUFFERED=1
120
 
121
  EXPOSE 7860
122
 
@@ -147,6 +139,20 @@ EXPOSE 7860
147
  # batches worth multiplying (was 64/32). RAM headroom (~16 GB vs ~5 GB used)
148
  # covers the larger compute buffers.
149
  # Validate any change with tests/bench.py (see PERFORMANCE_REFACTORING_PLAN.md).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
  ENV DEFAULT_MODEL=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
151
  MANAGER_HOST=0.0.0.0 \
152
  MANAGER_PORT=7860 \
 
18
  # Clone and build llama.cpp with SSL support for HuggingFace Hub
19
  WORKDIR /build
20
  ARG CACHEBUST=6
21
+ # Pinned release tag (supply chain: never build from a moving branch). Bump
22
+ # deliberately and re-run the endpoint tests.
23
+ ARG LLAMA_CPP_REF=b12367
24
  #ARG BUILD_PROFILE=fast_build
25
  ARG BUILD_PROFILE=fast_runtime
26
  ARG BUILD_JOBS=1
 
93
  # Update library cache
94
  RUN ldconfig
95
 
96
+ # The legacy Python app (python/) is intentionally NOT shipped: it has
97
+ # unauthenticated admin endpoints and a pile of unpinned dependencies. The
98
+ # runtime image contains only llama-server and the C++ manager.
 
 
 
 
 
99
 
100
  # Create non-root user
101
  RUN useradd -m -u 1000 user && \
102
  mkdir -p /home/user/.cache/llama.cpp && \
103
  chown -R user:user /home/user
104
 
 
 
 
105
  USER user
106
  WORKDIR /home/user
107
 
108
  # Set environment variables
109
  ENV HOME=/home/user \
110
  LLAMA_CACHE=/home/user/.cache/llama.cpp \
111
+ PATH=/home/user/.local/bin:$PATH
 
 
112
 
113
  EXPOSE 7860
114
 
 
139
  # batches worth multiplying (was 64/32). RAM headroom (~16 GB vs ~5 GB used)
140
  # covers the larger compute buffers.
141
  # Validate any change with tests/bench.py (see PERFORMANCE_REFACTORING_PLAN.md).
142
+ #
143
+ # Security (see docs/SECURITY_AUDIT_AND_FIX_PLAN.md):
144
+ # - API keys are NOT baked into the image. Set the API_KEYS secret on the
145
+ # Space: API_KEYS="admin-main:admin:<32+ char secret>,user-main:user:<secret>"
146
+ # (a secret may also be given as sha256:<hex>). Without it the manager
147
+ # fails closed and every authenticated endpoint returns 401.
148
+ # - ALLOWED_MODELS restricts /switch-model; the default model is always allowed.
149
+ # - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
150
+ # (X-Forwarded-For from the HF proxy).
151
+ ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
152
+ PER_IP_REQUESTS_PER_MINUTE=120 \
153
+ MANAGER_MAX_SESSIONS=256 \
154
+ MAX_REQUEST_BODY_BYTES=1048576 \
155
+ LOG_REQUEST_BODIES=false
156
  ENV DEFAULT_MODEL=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
157
  MANAGER_HOST=0.0.0.0 \
158
  MANAGER_PORT=7860 \
README.md CHANGED
@@ -25,6 +25,38 @@ The current primary path in this repository is the C++ `llm-manager`:
25
 
26
  The legacy Python application is still present under [`python/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/python), but the repository is now centered around the C++ manager implementation in [`cpp/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/cpp).
27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  ## Features
29
 
30
  - Dynamic model switching through a single manager endpoint.
 
25
 
26
  The legacy Python application is still present under [`python/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/python), but the repository is now centered around the C++ manager implementation in [`cpp/`](/Users/dmitryberesnev/Project/huggingface/agi/AGI/cpp).
27
 
28
+ ## Security Setup (read before deploying)
29
+
30
+ The manager **fails closed**: with no usable API keys configured every authenticated endpoint
31
+ (`/v1/chat/completions`, `/switch-model`, `/stop`, cancel, `/props`, `/slots`, `/metrics`) returns 401.
32
+ Keys are never baked into the image. Configure them as an environment variable / HF Space secret:
33
+
34
+ ```bash
35
+ # key_id:role:secret[,key_id:role:secret...]; role is admin or user
36
+ API_KEYS="admin-main:admin:$(openssl rand -hex 24),user-main:user:$(openssl rand -hex 24)"
37
+ # a secret may also be given as sha256:<hex> so the plaintext is never stored
38
+ ```
39
+
40
+ Rules enforced at startup (violations disable the key and are logged):
41
+
42
+ - secrets must be at least 32 characters (`AUTH_MIN_SECRET_LENGTH`)
43
+ - placeholder secrets (`change-me…`, `test…`, `replace…`, …) are refused
44
+ - `AUTH_ALLOW_ANONYMOUS=true` lets unauthenticated callers in with the **user** role only
45
+
46
+ Other security-relevant settings (all overridable by environment variable):
47
+
48
+ | Setting | Env | Purpose |
49
+ |---------|-----|---------|
50
+ | `[models] allowed` | `ALLOWED_MODELS` | Allowlist for `/switch-model`; default model always allowed |
51
+ | `[rate_limit] per_ip_requests_per_minute` | `PER_IP_REQUESTS_PER_MINUTE` | Pre-auth per-address limit (uses `X-Forwarded-For` when `trust_proxy_headers`) |
52
+ | `[server] max_sessions` | `MANAGER_MAX_SESSIONS` | Connection cap; excess gets a fast 503 |
53
+ | `[limits] max_request_body_bytes` | `MAX_REQUEST_BODY_BYTES` | 413 for oversized bodies |
54
+ | `[logging] log_request_bodies` | `LOG_REQUEST_BODIES` | Prompts stay out of logs unless enabled |
55
+
56
+ Cancellation is soft: cancelling a running request closes only that request's upstream connection.
57
+ Only the admin `/stop` endpoint restarts the worker. The full audit and rationale is in
58
+ `docs/SECURITY_AUDIT_AND_FIX_PLAN.md`.
59
+
60
  ## Features
61
 
62
  - Dynamic model switching through a single manager endpoint.
config.toml.example CHANGED
@@ -1,6 +1,10 @@
1
  [server]
2
  host = "0.0.0.0"
3
  port = 7860
 
 
 
 
4
 
5
  [worker]
6
  default_model = "QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"
@@ -25,11 +29,18 @@ ubatch = 128
25
  [auth]
26
  header = "Authorization"
27
  scheme = "Bearer"
 
 
 
 
 
28
 
29
  [limits]
30
  default_max_tokens = 256
31
  max_tokens_per_request = 2048
32
  request_timeout_sec = 30
 
 
33
 
34
  [queue]
35
  max_size = 100
@@ -46,15 +57,36 @@ enabled = false
46
  [rate_limit]
47
  requests_per_minute = 60
48
  estimated_tokens_per_minute = 6000
 
 
 
 
 
49
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  [[api_keys]]
51
  key_id = "admin-main"
52
- secret = "change-me-admin"
 
53
  role = "admin"
54
  enabled = true
55
 
56
  [[api_keys]]
57
  key_id = "user-main"
58
- secret = "change-me-user"
 
59
  role = "user"
60
  enabled = true
 
1
  [server]
2
  host = "0.0.0.0"
3
  port = 7860
4
+ # Max concurrently open client connections (each pins a thread); excess get 503.
5
+ max_sessions = 256
6
+ # Per-recv timeout for client sockets (idle keep-alive, slow senders).
7
+ read_timeout_sec = 30
8
 
9
  [worker]
10
  default_model = "QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"
 
29
  [auth]
30
  header = "Authorization"
31
  scheme = "Bearer"
32
+ # Fail closed: with no usable api keys every authenticated endpoint is 401.
33
+ # Set true to let unauthenticated callers in with the "user" role (never admin).
34
+ allow_anonymous = false
35
+ # Plaintext secrets shorter than this are refused at load time.
36
+ min_secret_length = 32
37
 
38
  [limits]
39
  default_max_tokens = 256
40
  max_tokens_per_request = 2048
41
  request_timeout_sec = 30
42
+ # Bodies above this get 413 before parsing.
43
+ max_request_body_bytes = 1048576
44
 
45
  [queue]
46
  max_size = 100
 
57
  [rate_limit]
58
  requests_per_minute = 60
59
  estimated_tokens_per_minute = 6000
60
+ # Pre-auth limit per client address, applied to every request (0 = off).
61
+ per_ip_requests_per_minute = 120
62
+ # Use the first X-Forwarded-For hop as the client address (true behind the
63
+ # HF proxy or any trusted reverse proxy; false if clients connect directly).
64
+ trust_proxy_headers = true
65
 
66
+ [logging]
67
+ # Request bodies contain user prompts; only enable for local debugging.
68
+ log_request_bodies = false
69
+
70
+ [models]
71
+ # Models /switch-model may load (the default model is always allowed).
72
+ allowed = ["QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"]
73
+
74
+ # API keys. Prefer the API_KEYS environment variable (HF Space secret) in
75
+ # production: "key_id:role:secret,key_id:role:secret". When API_KEYS is set
76
+ # the entries below are ignored. Placeholder secrets ("change-me...", "test",
77
+ # ...) and secrets shorter than min_secret_length are disabled at load time.
78
+ # Generate one with: openssl rand -hex 24
79
+ # Hash one with: printf '%s' "$SECRET" | sha256sum
80
  [[api_keys]]
81
  key_id = "admin-main"
82
+ # Either a plaintext secret ...
83
+ secret = "REPLACE-WITH-32-PLUS-RANDOM-CHARS-admin"
84
  role = "admin"
85
  enabled = true
86
 
87
  [[api_keys]]
88
  key_id = "user-main"
89
+ # ... or only its SHA-256 so the plaintext never sits in the config file.
90
+ secret_sha256 = "0000000000000000000000000000000000000000000000000000000000000000"
91
  role = "user"
92
  enabled = true
cpp/config.cpp CHANGED
@@ -1,12 +1,16 @@
1
  #include "config.h"
2
 
3
  #include "http_helpers.h"
 
4
 
5
  #include <algorithm>
 
6
  #include <cstdlib>
7
  #include <filesystem>
8
  #include <fstream>
 
9
  #include <unordered_map>
 
10
 
11
  std::string get_env_or(const char *name, const std::string &fallback) {
12
  const char *v = std::getenv(name);
@@ -112,7 +116,7 @@ static std::vector<ApiKeyRecord> parse_api_keys_toml(const std::string &path) {
112
  bool has_any_field = false;
113
 
114
  auto flush_current = [&]() {
115
- if (has_any_field && !current.key_id.empty() && !current.secret.empty()) {
116
  keys.push_back(current);
117
  }
118
  current = ApiKeyRecord{};
@@ -145,6 +149,7 @@ static std::vector<ApiKeyRecord> parse_api_keys_toml(const std::string &path) {
145
 
146
  if (key == "key_id") current.key_id = value;
147
  else if (key == "secret") current.secret = value;
 
148
  else if (key == "role") current.role = parse_role_or(value, current.role);
149
  else if (key == "enabled") current.enabled = parse_bool_or(value, current.enabled);
150
  }
@@ -181,6 +186,112 @@ static int get_toml_int_or(
181
  }
182
  }
183
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
184
  ManagerConfig load_manager_config() {
185
  ManagerConfig cfg;
186
 
@@ -195,6 +306,8 @@ ManagerConfig load_manager_config() {
195
 
196
  cfg.server.host = get_env_or("MANAGER_HOST", get_toml_string_or(toml, "server", "host", cfg.server.host));
197
  cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
 
 
198
 
199
  cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
200
  cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
@@ -211,10 +324,15 @@ ManagerConfig load_manager_config() {
211
 
212
  cfg.auth.header = get_env_or("AUTH_HEADER", get_toml_string_or(toml, "auth", "header", cfg.auth.header));
213
  cfg.auth.scheme = get_env_or("AUTH_SCHEME", get_toml_string_or(toml, "auth", "scheme", cfg.auth.scheme));
 
 
 
 
214
 
215
  cfg.limits.default_max_tokens = get_env_int_or("DEFAULT_MAX_TOKENS", get_toml_int_or(toml, "limits", "default_max_tokens", cfg.limits.default_max_tokens));
216
  cfg.limits.max_tokens_per_request = get_env_int_or("MAX_TOKENS_PER_REQUEST", get_toml_int_or(toml, "limits", "max_tokens_per_request", cfg.limits.max_tokens_per_request));
217
  cfg.limits.request_timeout_sec = get_env_int_or("REQUEST_TIMEOUT_SEC", get_toml_int_or(toml, "limits", "request_timeout_sec", cfg.limits.request_timeout_sec));
 
218
 
219
  cfg.queue.max_size = static_cast<size_t>(std::max(
220
  1,
@@ -225,15 +343,39 @@ ManagerConfig load_manager_config() {
225
 
226
  cfg.rate_limit.requests_per_minute = get_env_int_or("REQUESTS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "requests_per_minute", cfg.rate_limit.requests_per_minute));
227
  cfg.rate_limit.estimated_tokens_per_minute = get_env_int_or("ESTIMATED_TOKENS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "estimated_tokens_per_minute", cfg.rate_limit.estimated_tokens_per_minute));
 
 
 
 
 
 
 
228
 
229
  cfg.scheduler.max_concurrent = get_env_int_or("SCHEDULER_MAX_CONCURRENT", get_toml_int_or(toml, "scheduler", "max_concurrent", cfg.scheduler.max_concurrent));
230
  cfg.streaming.enabled = parse_bool_or(
231
  get_env_or("STREAMING_ENABLED", get_toml_string_or(toml, "streaming", "enabled", cfg.streaming.enabled ? "true" : "false")),
232
  cfg.streaming.enabled);
233
 
234
- if (!config_path.empty() && std::filesystem::exists(config_path)) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  cfg.api_keys = parse_api_keys_toml(config_path);
236
  }
 
237
 
238
  return cfg;
239
  }
 
1
  #include "config.h"
2
 
3
  #include "http_helpers.h"
4
+ #include "sha256.h"
5
 
6
  #include <algorithm>
7
+ #include <cctype>
8
  #include <cstdlib>
9
  #include <filesystem>
10
  #include <fstream>
11
+ #include <sstream>
12
  #include <unordered_map>
13
+ #include <unordered_set>
14
 
15
  std::string get_env_or(const char *name, const std::string &fallback) {
16
  const char *v = std::getenv(name);
 
116
  bool has_any_field = false;
117
 
118
  auto flush_current = [&]() {
119
+ if (has_any_field && !current.key_id.empty() && (!current.secret.empty() || !current.secret_sha256.empty())) {
120
  keys.push_back(current);
121
  }
122
  current = ApiKeyRecord{};
 
149
 
150
  if (key == "key_id") current.key_id = value;
151
  else if (key == "secret") current.secret = value;
152
+ else if (key == "secret_sha256") current.secret_sha256 = value;
153
  else if (key == "role") current.role = parse_role_or(value, current.role);
154
  else if (key == "enabled") current.enabled = parse_bool_or(value, current.enabled);
155
  }
 
186
  }
187
  }
188
 
189
+ // Splits a comma-separated list, trimming and dropping empty entries.
190
+ static std::vector<std::string> split_csv(const std::string &value) {
191
+ std::vector<std::string> out;
192
+ std::stringstream ss(value);
193
+ std::string item;
194
+ while (std::getline(ss, item, ',')) {
195
+ item = strip_quotes(trim_copy(item));
196
+ if (!item.empty()) out.push_back(item);
197
+ }
198
+ return out;
199
+ }
200
+
201
+ // Parses a one-line TOML string array: allowed = ["a", "b"]. The simple
202
+ // parser stores the raw right-hand side, brackets included.
203
+ static std::vector<std::string> parse_toml_string_array(const std::string &raw) {
204
+ std::string inner = trim_copy(raw);
205
+ if (inner.size() >= 2 && inner.front() == '[' && inner.back() == ']') {
206
+ inner = inner.substr(1, inner.size() - 2);
207
+ }
208
+ return split_csv(inner);
209
+ }
210
+
211
+ // API_KEYS="key_id:role:secret,key_id:role:secret". A secret may contain ':'
212
+ // (only the first two separators are significant) and may be given as
213
+ // "sha256:<hex>" to avoid putting the plaintext in the environment.
214
+ static std::vector<ApiKeyRecord> parse_api_keys_env(const std::string &raw) {
215
+ std::vector<ApiKeyRecord> keys;
216
+ for (const auto &entry : split_csv(raw)) {
217
+ const auto first = entry.find(':');
218
+ if (first == std::string::npos) {
219
+ log_line("config: ignoring malformed API_KEYS entry (expected key_id:role:secret)");
220
+ continue;
221
+ }
222
+ const auto second = entry.find(':', first + 1);
223
+ if (second == std::string::npos) {
224
+ log_line("config: ignoring malformed API_KEYS entry (expected key_id:role:secret)");
225
+ continue;
226
+ }
227
+ ApiKeyRecord record;
228
+ record.key_id = trim_copy(entry.substr(0, first));
229
+ record.role = parse_role_or(entry.substr(first + 1, second - first - 1), Role::USER);
230
+ std::string secret = trim_copy(entry.substr(second + 1));
231
+ if (secret.rfind("sha256:", 0) == 0) {
232
+ record.secret_sha256 = secret.substr(7);
233
+ } else {
234
+ record.secret = secret;
235
+ }
236
+ if (record.key_id.empty() || (record.secret.empty() && record.secret_sha256.empty())) {
237
+ log_line("config: ignoring API_KEYS entry with empty key_id or secret");
238
+ continue;
239
+ }
240
+ keys.push_back(record);
241
+ }
242
+ return keys;
243
+ }
244
+
245
+ static bool is_hex_digest(const std::string &value) {
246
+ if (value.size() != 64) return false;
247
+ return std::all_of(value.begin(), value.end(), [](unsigned char c) {
248
+ return std::isxdigit(c) != 0;
249
+ });
250
+ }
251
+
252
+ // Normalises every record to a lowercase hex digest, refuses placeholder and
253
+ // short secrets, and drops duplicates. Refused keys are disabled rather than
254
+ // silently accepted so a misconfigured deployment fails closed.
255
+ static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig &auth) {
256
+ static const char *kPlaceholders[] = {"change-me", "changeme", "replace", "secret", "password", "test", "example"};
257
+ std::unordered_set<std::string> seen_ids;
258
+ for (auto &record : keys) {
259
+ if (!seen_ids.insert(record.key_id).second) {
260
+ log_line("config: duplicate api key id '" + record.key_id + "' — disabling later entry");
261
+ record.enabled = false;
262
+ continue;
263
+ }
264
+ if (!record.secret.empty()) {
265
+ std::string lowered = record.secret;
266
+ std::transform(lowered.begin(), lowered.end(), lowered.begin(), [](unsigned char c) {
267
+ return static_cast<char>(std::tolower(c));
268
+ });
269
+ for (const char *placeholder : kPlaceholders) {
270
+ if (lowered.rfind(placeholder, 0) == 0) {
271
+ log_line("config: api key '" + record.key_id + "' uses a placeholder secret — disabled");
272
+ record.enabled = false;
273
+ break;
274
+ }
275
+ }
276
+ if (record.enabled && static_cast<int>(record.secret.size()) < auth.min_secret_length) {
277
+ log_line("config: api key '" + record.key_id + "' secret shorter than " +
278
+ std::to_string(auth.min_secret_length) + " chars — disabled");
279
+ record.enabled = false;
280
+ }
281
+ record.secret_sha256 = sha256_hex(record.secret);
282
+ // The plaintext is not needed after hashing; drop it from memory.
283
+ record.secret.clear();
284
+ } else {
285
+ std::transform(record.secret_sha256.begin(), record.secret_sha256.end(), record.secret_sha256.begin(),
286
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
287
+ if (!is_hex_digest(record.secret_sha256)) {
288
+ log_line("config: api key '" + record.key_id + "' has an invalid secret_sha256 — disabled");
289
+ record.enabled = false;
290
+ }
291
+ }
292
+ }
293
+ }
294
+
295
  ManagerConfig load_manager_config() {
296
  ManagerConfig cfg;
297
 
 
306
 
307
  cfg.server.host = get_env_or("MANAGER_HOST", get_toml_string_or(toml, "server", "host", cfg.server.host));
308
  cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
309
+ cfg.server.max_sessions = std::max(1, get_env_int_or("MANAGER_MAX_SESSIONS", get_toml_int_or(toml, "server", "max_sessions", cfg.server.max_sessions)));
310
+ cfg.server.read_timeout_sec = std::max(1, get_env_int_or("MANAGER_READ_TIMEOUT_SEC", get_toml_int_or(toml, "server", "read_timeout_sec", cfg.server.read_timeout_sec)));
311
 
312
  cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
313
  cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
 
324
 
325
  cfg.auth.header = get_env_or("AUTH_HEADER", get_toml_string_or(toml, "auth", "header", cfg.auth.header));
326
  cfg.auth.scheme = get_env_or("AUTH_SCHEME", get_toml_string_or(toml, "auth", "scheme", cfg.auth.scheme));
327
+ cfg.auth.allow_anonymous = parse_bool_or(
328
+ get_env_or("AUTH_ALLOW_ANONYMOUS", get_toml_string_or(toml, "auth", "allow_anonymous", cfg.auth.allow_anonymous ? "true" : "false")),
329
+ cfg.auth.allow_anonymous);
330
+ cfg.auth.min_secret_length = std::max(1, get_env_int_or("AUTH_MIN_SECRET_LENGTH", get_toml_int_or(toml, "auth", "min_secret_length", cfg.auth.min_secret_length)));
331
 
332
  cfg.limits.default_max_tokens = get_env_int_or("DEFAULT_MAX_TOKENS", get_toml_int_or(toml, "limits", "default_max_tokens", cfg.limits.default_max_tokens));
333
  cfg.limits.max_tokens_per_request = get_env_int_or("MAX_TOKENS_PER_REQUEST", get_toml_int_or(toml, "limits", "max_tokens_per_request", cfg.limits.max_tokens_per_request));
334
  cfg.limits.request_timeout_sec = get_env_int_or("REQUEST_TIMEOUT_SEC", get_toml_int_or(toml, "limits", "request_timeout_sec", cfg.limits.request_timeout_sec));
335
+ cfg.limits.max_request_body_bytes = std::max(1024, get_env_int_or("MAX_REQUEST_BODY_BYTES", get_toml_int_or(toml, "limits", "max_request_body_bytes", cfg.limits.max_request_body_bytes)));
336
 
337
  cfg.queue.max_size = static_cast<size_t>(std::max(
338
  1,
 
343
 
344
  cfg.rate_limit.requests_per_minute = get_env_int_or("REQUESTS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "requests_per_minute", cfg.rate_limit.requests_per_minute));
345
  cfg.rate_limit.estimated_tokens_per_minute = get_env_int_or("ESTIMATED_TOKENS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "estimated_tokens_per_minute", cfg.rate_limit.estimated_tokens_per_minute));
346
+ cfg.rate_limit.per_ip_requests_per_minute = get_env_int_or("PER_IP_REQUESTS_PER_MINUTE", get_toml_int_or(toml, "rate_limit", "per_ip_requests_per_minute", cfg.rate_limit.per_ip_requests_per_minute));
347
+ cfg.rate_limit.trust_proxy_headers = parse_bool_or(
348
+ get_env_or("TRUST_PROXY_HEADERS", get_toml_string_or(toml, "rate_limit", "trust_proxy_headers", cfg.rate_limit.trust_proxy_headers ? "true" : "false")),
349
+ cfg.rate_limit.trust_proxy_headers);
350
+ cfg.logging.log_request_bodies = parse_bool_or(
351
+ get_env_or("LOG_REQUEST_BODIES", get_toml_string_or(toml, "logging", "log_request_bodies", cfg.logging.log_request_bodies ? "true" : "false")),
352
+ cfg.logging.log_request_bodies);
353
 
354
  cfg.scheduler.max_concurrent = get_env_int_or("SCHEDULER_MAX_CONCURRENT", get_toml_int_or(toml, "scheduler", "max_concurrent", cfg.scheduler.max_concurrent));
355
  cfg.streaming.enabled = parse_bool_or(
356
  get_env_or("STREAMING_ENABLED", get_toml_string_or(toml, "streaming", "enabled", cfg.streaming.enabled ? "true" : "false")),
357
  cfg.streaming.enabled);
358
 
359
+ // Model allowlist: env overrides file; the default model is always allowed.
360
+ const std::string allowed_env = get_env_or("ALLOWED_MODELS", "");
361
+ if (!allowed_env.empty()) {
362
+ cfg.models.allowed = split_csv(allowed_env);
363
+ } else {
364
+ cfg.models.allowed = parse_toml_string_array(get_toml_string_or(toml, "models", "allowed", ""));
365
+ }
366
+ if (std::find(cfg.models.allowed.begin(), cfg.models.allowed.end(), cfg.worker.default_model) == cfg.models.allowed.end()) {
367
+ cfg.models.allowed.push_back(cfg.worker.default_model);
368
+ }
369
+
370
+ // API keys: the environment (HF Space secrets) takes precedence over the
371
+ // file so a committed config can never override deployed credentials.
372
+ const std::string api_keys_env = get_env_or("API_KEYS", "");
373
+ if (!api_keys_env.empty()) {
374
+ cfg.api_keys = parse_api_keys_env(api_keys_env);
375
+ } else if (!config_path.empty() && std::filesystem::exists(config_path)) {
376
  cfg.api_keys = parse_api_keys_toml(config_path);
377
  }
378
+ finalize_api_keys(cfg.api_keys, cfg.auth);
379
 
380
  return cfg;
381
  }
cpp/http_helpers.cpp CHANGED
@@ -4,10 +4,19 @@
4
  #include <boost/beast/core.hpp>
5
  #include <boost/beast/http.hpp>
6
 
 
 
7
  #include <ctime>
8
  #include <iomanip>
9
  #include <iostream>
 
 
10
  #include <sstream>
 
 
 
 
 
11
 
12
  namespace asio = boost::asio;
13
  namespace beast = boost::beast;
@@ -23,7 +32,91 @@ std::string now_utc_iso() {
23
  }
24
 
25
  void log_line(const std::string &line) {
26
- std::cout << "[" << now_utc_iso() << "] " << line << std::endl;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  }
28
 
29
  std::string truncate_body(const std::string &body, size_t max_len) {
@@ -42,18 +135,34 @@ std::optional<std::string> extract_cancel_request_id(const std::string &path) {
42
  return request_id;
43
  }
44
 
45
- std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::string &body) {
46
  asio::io_context ioc;
47
  asio::ip::tcp::resolver resolver(ioc);
48
  beast::tcp_stream stream(ioc);
49
  auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
50
  stream.connect(results);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
  http::request<http::string_body> req{http::verb::post, "/v1/chat/completions", 11};
53
  req.set(http::field::host, "127.0.0.1");
54
  req.set(http::field::content_type, "application/json");
55
  req.set(http::field::user_agent, "llm-manager");
56
- req.body() = body;
57
  req.prepare_payload();
58
  http::write(stream, req);
59
 
@@ -71,6 +180,7 @@ ProxiedGetResult forward_get_to_worker(const WorkerInfo &worker, const std::stri
71
  beast::tcp_stream stream(ioc);
72
  auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
73
  stream.connect(results);
 
74
 
75
  http::request<http::string_body> req{http::verb::get, target, 11};
76
  req.set(http::field::host, "127.0.0.1");
 
4
  #include <boost/beast/core.hpp>
5
  #include <boost/beast/http.hpp>
6
 
7
+ #include <cctype>
8
+ #include <cstdio>
9
  #include <ctime>
10
  #include <iomanip>
11
  #include <iostream>
12
+ #include <mutex>
13
+ #include <random>
14
  #include <sstream>
15
+ #include <stdexcept>
16
+ #include <vector>
17
+
18
+ #include <sys/socket.h>
19
+ #include <sys/time.h>
20
 
21
  namespace asio = boost::asio;
22
  namespace beast = boost::beast;
 
32
  }
33
 
34
  void log_line(const std::string &line) {
35
+ std::string safe;
36
+ safe.reserve(line.size());
37
+ for (unsigned char c : line) {
38
+ if (c == '\n') safe += "\\n";
39
+ else if (c == '\r') safe += "\\r";
40
+ else if (c == '\t') safe += "\\t";
41
+ else if (c < 0x20 || c == 0x7f) {
42
+ char buf[8];
43
+ std::snprintf(buf, sizeof(buf), "\\x%02x", c);
44
+ safe += buf;
45
+ } else safe.push_back(static_cast<char>(c));
46
+ }
47
+ std::cout << "[" << now_utc_iso() << "] " << safe << std::endl;
48
+ }
49
+
50
+ void set_socket_timeouts(int fd, int recv_sec, int send_sec) {
51
+ timeval rcv{};
52
+ rcv.tv_sec = recv_sec;
53
+ setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast<const char *>(&rcv), sizeof(rcv));
54
+ timeval snd{};
55
+ snd.tv_sec = send_sec;
56
+ setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char *>(&snd), sizeof(snd));
57
+ }
58
+
59
+ std::string generate_request_id() {
60
+ static std::mutex mu;
61
+ static std::mt19937_64 rng = [] {
62
+ std::random_device rd;
63
+ std::seed_seq seed{rd(), rd(), rd(), rd(), rd(), rd(), rd(), rd()};
64
+ return std::mt19937_64(seed);
65
+ }();
66
+ uint64_t hi, lo;
67
+ {
68
+ std::lock_guard<std::mutex> lock(mu);
69
+ hi = rng();
70
+ lo = rng();
71
+ }
72
+ char buf[33];
73
+ std::snprintf(buf, sizeof(buf), "%016llx%016llx",
74
+ static_cast<unsigned long long>(hi), static_cast<unsigned long long>(lo));
75
+ return std::string(buf);
76
+ }
77
+
78
+ std::optional<std::string> normalize_path(const std::string &raw_path) {
79
+ // Percent-decode first: llama-server (cpp-httplib) decodes before routing,
80
+ // so the manager must compare the same form it will be routed as.
81
+ std::string decoded;
82
+ decoded.reserve(raw_path.size());
83
+ for (size_t i = 0; i < raw_path.size(); ++i) {
84
+ const char c = raw_path[i];
85
+ if (c == '%') {
86
+ if (i + 2 >= raw_path.size() ||
87
+ !std::isxdigit(static_cast<unsigned char>(raw_path[i + 1])) ||
88
+ !std::isxdigit(static_cast<unsigned char>(raw_path[i + 2]))) {
89
+ return std::nullopt;
90
+ }
91
+ decoded.push_back(static_cast<char>(std::stoi(raw_path.substr(i + 1, 2), nullptr, 16)));
92
+ i += 2;
93
+ } else {
94
+ decoded.push_back(c);
95
+ }
96
+ }
97
+ for (unsigned char c : decoded) {
98
+ if (c < 0x20 || c == 0x7f || c == '\\') return std::nullopt;
99
+ }
100
+ if (decoded.empty() || decoded.front() != '/') return std::nullopt;
101
+
102
+ std::vector<std::string> segments;
103
+ std::stringstream ss(decoded);
104
+ std::string seg;
105
+ while (std::getline(ss, seg, '/')) {
106
+ if (seg.empty() || seg == ".") continue;
107
+ if (seg == "..") {
108
+ if (segments.empty()) return std::nullopt;
109
+ segments.pop_back();
110
+ continue;
111
+ }
112
+ segments.push_back(seg);
113
+ }
114
+ std::string out = "/";
115
+ for (size_t i = 0; i < segments.size(); ++i) {
116
+ if (i) out += "/";
117
+ out += segments[i];
118
+ }
119
+ return out;
120
  }
121
 
122
  std::string truncate_body(const std::string &body, size_t max_len) {
 
135
  return request_id;
136
  }
137
 
138
+ std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::shared_ptr<RequestContext> &ctx) {
139
  asio::io_context ioc;
140
  asio::ip::tcp::resolver resolver(ioc);
141
  beast::tcp_stream stream(ioc);
142
  auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
143
  stream.connect(results);
144
+ // Generation can legitimately take minutes; the client-side request
145
+ // timeout cancels (and thereby unblocks) this read long before this
146
+ // safety net fires.
147
+ set_socket_timeouts(stream.socket().native_handle(), 3600, 60);
148
+
149
+ // Register the abort hook: a cancel from another thread shuts the socket
150
+ // down, which makes the blocking read below fail. The guard disarms it
151
+ // before the socket is destroyed so the hook can never touch a dead fd.
152
+ const int fd = stream.socket().native_handle();
153
+ struct AbortGuard {
154
+ const std::shared_ptr<RequestContext> &ctx;
155
+ ~AbortGuard() { ctx->disarm_abort(); }
156
+ } abort_guard{ctx};
157
+ if (!ctx->arm_abort([fd]() { ::shutdown(fd, SHUT_RDWR); })) {
158
+ throw std::runtime_error("request cancelled before upstream call");
159
+ }
160
 
161
  http::request<http::string_body> req{http::verb::post, "/v1/chat/completions", 11};
162
  req.set(http::field::host, "127.0.0.1");
163
  req.set(http::field::content_type, "application/json");
164
  req.set(http::field::user_agent, "llm-manager");
165
+ req.body() = ctx->request_body;
166
  req.prepare_payload();
167
  http::write(stream, req);
168
 
 
180
  beast::tcp_stream stream(ioc);
181
  auto const results = resolver.resolve("127.0.0.1", std::to_string(worker.port));
182
  stream.connect(results);
183
+ set_socket_timeouts(stream.socket().native_handle(), 30, 30);
184
 
185
  http::request<http::string_body> req{http::verb::get, target, 11};
186
  req.set(http::field::host, "127.0.0.1");
cpp/http_helpers.h CHANGED
@@ -7,8 +7,20 @@
7
  #include "llm_manager_types.h"
8
 
9
  std::string now_utc_iso();
 
 
10
  void log_line(const std::string &line);
11
  std::string truncate_body(const std::string &body, size_t max_len = 2000);
12
  std::optional<std::string> extract_cancel_request_id(const std::string &path);
13
- std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::string &body);
 
 
 
 
 
 
 
 
14
  ProxiedGetResult forward_get_to_worker(const WorkerInfo &worker, const std::string &target);
 
 
 
7
  #include "llm_manager_types.h"
8
 
9
  std::string now_utc_iso();
10
+ // Writes one log line; control characters in `line` are escaped so a client
11
+ // cannot forge log entries.
12
  void log_line(const std::string &line);
13
  std::string truncate_body(const std::string &body, size_t max_len = 2000);
14
  std::optional<std::string> extract_cancel_request_id(const std::string &path);
15
+ // Percent-decodes and canonicalises a URL path (collapses "//", resolves "."
16
+ // and ".."). Returns nullopt for paths that escape the root or contain
17
+ // control characters, so callers can match against an exact allowlist.
18
+ std::optional<std::string> normalize_path(const std::string &raw_path);
19
+ // Generates a 128-bit random request id as lowercase hex.
20
+ std::string generate_request_id();
21
+ // Forwards the request to the worker. While the upstream connection is open
22
+ // it is registered on `ctx` so a concurrent cancel can shut it down.
23
+ std::pair<int, std::string> forward_chat(const WorkerInfo &worker, const std::shared_ptr<RequestContext> &ctx);
24
  ProxiedGetResult forward_get_to_worker(const WorkerInfo &worker, const std::string &target);
25
+ // Applies SO_RCVTIMEO/SO_SNDTIMEO to a native socket.
26
+ void set_socket_timeouts(int fd, int recv_sec, int send_sec);
cpp/llm_manager.cpp CHANGED
@@ -1,5 +1,6 @@
1
  #include <boost/asio.hpp>
2
 
 
3
  #include <chrono>
4
  #include <functional>
5
  #include <string>
@@ -22,15 +23,30 @@ int main() {
22
  ModelManager manager(config);
23
  ApiKeyAuth auth(config);
24
  RateLimiterStore rate_limiter(config.rate_limit);
 
 
 
 
25
  RequestRegistry registry;
26
  MetricsRegistry metrics;
27
  Scheduler scheduler(manager, registry, metrics, config.queue);
 
 
28
 
29
  std::string init_error;
30
  if (auth.enabled()) {
31
- log_line("auth: enabled api_keys=" + std::to_string(config.api_keys.size()));
 
 
 
 
 
32
  } else {
33
- log_line("auth: disabled (no configured api keys)");
 
 
 
 
34
  }
35
 
36
  log_line("startup: loading default model");
@@ -57,15 +73,28 @@ int main() {
57
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
58
  continue;
59
  }
60
- std::thread(
61
- &do_session,
62
- std::move(socket),
63
- std::ref(manager),
64
- std::cref(config),
65
- std::cref(auth),
66
- std::ref(rate_limiter),
67
- std::ref(registry),
68
- std::ref(metrics),
69
- std::ref(scheduler)).detach();
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  }
71
  }
 
1
  #include <boost/asio.hpp>
2
 
3
+ #include <atomic>
4
  #include <chrono>
5
  #include <functional>
6
  #include <string>
 
23
  ModelManager manager(config);
24
  ApiKeyAuth auth(config);
25
  RateLimiterStore rate_limiter(config.rate_limit);
26
+ RateLimitConfig ip_limit_config;
27
+ ip_limit_config.requests_per_minute = config.rate_limit.per_ip_requests_per_minute;
28
+ ip_limit_config.estimated_tokens_per_minute = 0;
29
+ RateLimiterStore ip_rate_limiter(ip_limit_config);
30
  RequestRegistry registry;
31
  MetricsRegistry metrics;
32
  Scheduler scheduler(manager, registry, metrics, config.queue);
33
+ std::atomic<int> active_sessions{0};
34
+ ServerDeps deps{manager, config, auth, rate_limiter, ip_rate_limiter, registry, metrics, scheduler, active_sessions};
35
 
36
  std::string init_error;
37
  if (auth.enabled()) {
38
+ size_t usable = 0;
39
+ for (const auto &k : config.api_keys) if (k.enabled) ++usable;
40
+ log_line("auth: enabled api_keys=" + std::to_string(usable) + "/" + std::to_string(config.api_keys.size()) +
41
+ (config.auth.allow_anonymous ? " allow_anonymous=true(role=user)" : ""));
42
+ } else if (config.auth.allow_anonymous) {
43
+ log_line("auth: WARNING no usable api keys; anonymous access enabled with USER role");
44
  } else {
45
+ log_line("auth: no usable api keys configured — all authenticated endpoints will return 401 "
46
+ "(set API_KEYS or [[api_keys]], or AUTH_ALLOW_ANONYMOUS=true)");
47
+ }
48
+ if (config.models.allowed.size() <= 1) {
49
+ log_line("models: allowlist contains only the default model (set ALLOWED_MODELS to permit others)");
50
  }
51
 
52
  log_line("startup: loading default model");
 
73
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
74
  continue;
75
  }
76
+ // Connection cap: each session pins a thread, so refuse fast once the
77
+ // configured maximum is reached instead of letting a flood exhaust us.
78
+ if (active_sessions.load() >= config.server.max_sessions) {
79
+ static const std::string kBusy =
80
+ "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
81
+ "Retry-After: 2\r\nConnection: close\r\nContent-Length: 29\r\n\r\n"
82
+ "{\"error\":\"Too many sessions\"}";
83
+ boost::system::error_code ignored;
84
+ asio::write(socket, asio::buffer(kBusy), ignored);
85
+ socket.shutdown(asio::ip::tcp::socket::shutdown_both, ignored);
86
+ socket.close(ignored);
87
+ continue;
88
+ }
89
+ active_sessions.fetch_add(1);
90
+ try {
91
+ std::thread(&do_session, std::move(socket), std::ref(deps)).detach();
92
+ } catch (const std::exception &e) {
93
+ // Thread creation failed (resource exhaustion): the socket was
94
+ // moved into the thread argument and is gone; just release the slot.
95
+ active_sessions.fetch_sub(1);
96
+ log_line("thread_spawn_error=" + std::string(e.what()));
97
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
98
+ }
99
  }
100
  }
cpp/llm_manager_types.h CHANGED
@@ -3,6 +3,7 @@
3
  #include <atomic>
4
  #include <chrono>
5
  #include <condition_variable>
 
6
  #include <memory>
7
  #include <mutex>
8
  #include <string>
@@ -17,6 +18,12 @@ using json = nlohmann::json;
17
  struct ServerConfig {
18
  std::string host = "0.0.0.0";
19
  int port = 7860;
 
 
 
 
 
 
20
  };
21
 
22
  struct WorkerConfig {
@@ -47,12 +54,20 @@ enum class Role {
47
  struct AuthConfig {
48
  std::string header = "Authorization";
49
  std::string scheme = "Bearer";
 
 
 
 
 
 
50
  };
51
 
52
  struct LimitsConfig {
53
  int default_max_tokens = 256;
54
  int max_tokens_per_request = 2048;
55
  int request_timeout_sec = 30;
 
 
56
  };
57
 
58
  struct QueueConfig {
@@ -65,6 +80,22 @@ struct QueueConfig {
65
  struct RateLimitConfig {
66
  int requests_per_minute = 60;
67
  int estimated_tokens_per_minute = 6000;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  };
69
 
70
  struct SchedulerConfig {
@@ -77,7 +108,11 @@ struct StreamingConfig {
77
 
78
  struct ApiKeyRecord {
79
  std::string key_id;
 
 
80
  std::string secret;
 
 
81
  Role role = Role::USER;
82
  bool enabled = true;
83
  };
@@ -92,6 +127,8 @@ struct ManagerConfig {
92
  RateLimitConfig rate_limit;
93
  SchedulerConfig scheduler;
94
  StreamingConfig streaming;
 
 
95
  std::vector<ApiKeyRecord> api_keys;
96
  };
97
 
@@ -116,6 +153,8 @@ enum class RequestState {
116
  };
117
 
118
  struct TokenEstimate {
 
 
119
  int prompt_tokens = 0;
120
  int requested_max_tokens = 0;
121
  int estimated_total_tokens = 0;
@@ -156,6 +195,30 @@ struct RequestContext {
156
  std::condition_variable cv;
157
  bool completed = false;
158
  RequestResult result;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  };
160
 
161
  struct ProxiedGetResult {
 
3
  #include <atomic>
4
  #include <chrono>
5
  #include <condition_variable>
6
+ #include <functional>
7
  #include <memory>
8
  #include <mutex>
9
  #include <string>
 
18
  struct ServerConfig {
19
  std::string host = "0.0.0.0";
20
  int port = 7860;
21
+ // Hard cap on concurrently open client connections (each one pins a
22
+ // thread). Excess connections get a fast 503.
23
+ int max_sessions = 256;
24
+ // Per-recv timeout for client sockets: bounds idle keep-alive connections
25
+ // and stalled senders.
26
+ int read_timeout_sec = 30;
27
  };
28
 
29
  struct WorkerConfig {
 
54
  struct AuthConfig {
55
  std::string header = "Authorization";
56
  std::string scheme = "Bearer";
57
+ // Fail closed by default: with no API keys configured every request is
58
+ // rejected. Setting this lets unauthenticated callers in with USER role
59
+ // (never ADMIN).
60
+ bool allow_anonymous = false;
61
+ // Secrets shorter than this are refused at load time.
62
+ int min_secret_length = 32;
63
  };
64
 
65
  struct LimitsConfig {
66
  int default_max_tokens = 256;
67
  int max_tokens_per_request = 2048;
68
  int request_timeout_sec = 30;
69
+ // Upper bound on any request body; larger bodies get 413 before parsing.
70
+ int max_request_body_bytes = 1 * 1024 * 1024;
71
  };
72
 
73
  struct QueueConfig {
 
80
  struct RateLimitConfig {
81
  int requests_per_minute = 60;
82
  int estimated_tokens_per_minute = 6000;
83
+ // Pre-auth per-source limit applied to every request (0 = disabled). The
84
+ // source is the first X-Forwarded-For hop when trust_proxy_headers is
85
+ // set, otherwise the TCP peer address.
86
+ int per_ip_requests_per_minute = 0;
87
+ bool trust_proxy_headers = true;
88
+ };
89
+
90
+ struct LoggingConfig {
91
+ // Request bodies carry user prompts; keep them out of logs unless
92
+ // explicitly enabled for debugging.
93
+ bool log_request_bodies = false;
94
+ };
95
+
96
+ struct ModelsConfig {
97
+ // Models /switch-model may load. Empty means only the default model.
98
+ std::vector<std::string> allowed;
99
  };
100
 
101
  struct SchedulerConfig {
 
108
 
109
  struct ApiKeyRecord {
110
  std::string key_id;
111
+ // Plaintext secret as loaded from config (may be empty when only the
112
+ // hash is configured). Never logged.
113
  std::string secret;
114
+ // Lowercase hex SHA-256 of the secret; this is what auth compares.
115
+ std::string secret_sha256;
116
  Role role = Role::USER;
117
  bool enabled = true;
118
  };
 
127
  RateLimitConfig rate_limit;
128
  SchedulerConfig scheduler;
129
  StreamingConfig streaming;
130
+ LoggingConfig logging;
131
+ ModelsConfig models;
132
  std::vector<ApiKeyRecord> api_keys;
133
  };
134
 
 
153
  };
154
 
155
  struct TokenEstimate {
156
+ // All values are validated to be non-negative and <= max_tokens_per_request
157
+ // before a request is admitted, so they always fit in int.
158
  int prompt_tokens = 0;
159
  int requested_max_tokens = 0;
160
  int estimated_total_tokens = 0;
 
195
  std::condition_variable cv;
196
  bool completed = false;
197
  RequestResult result;
198
+
199
+ // Soft cancellation: while the scheduler has an upstream connection open
200
+ // for this request it registers a callback that shuts that socket down.
201
+ // Cancelling then aborts only this request instead of restarting the
202
+ // worker (llama-server drops the task when the client disconnects).
203
+ std::mutex abort_mu;
204
+ std::function<void()> abort_upstream;
205
+
206
+ // Returns false if the request was already cancelled, in which case the
207
+ // caller must not proceed with the upstream call.
208
+ bool arm_abort(std::function<void()> fn) {
209
+ std::lock_guard<std::mutex> lock(abort_mu);
210
+ if (cancelled.load()) return false;
211
+ abort_upstream = std::move(fn);
212
+ return true;
213
+ }
214
+ void disarm_abort() {
215
+ std::lock_guard<std::mutex> lock(abort_mu);
216
+ abort_upstream = nullptr;
217
+ }
218
+ void trigger_abort() {
219
+ std::lock_guard<std::mutex> lock(abort_mu);
220
+ if (abort_upstream) abort_upstream();
221
+ }
222
  };
223
 
224
  struct ProxiedGetResult {
cpp/model_manager.cpp CHANGED
@@ -172,16 +172,14 @@ json ModelManager::models_view() {
172
  json out;
173
  out["status"] = (active_ && is_alive(active_->pid)) ? "ready" : "no_active_model";
174
  out["switch_in_progress"] = switch_in_progress_;
 
 
175
  if (active_ && is_alive(active_->pid)) {
176
  out["current_model"] = active_->model;
177
  out["last_loaded"] = active_->last_loaded;
178
- out["active_pid"] = active_->pid;
179
- out["active_port"] = active_->port;
180
  } else {
181
  out["current_model"] = nullptr;
182
  out["last_loaded"] = nullptr;
183
- out["active_pid"] = nullptr;
184
- out["active_port"] = nullptr;
185
  }
186
  return out;
187
  }
@@ -264,6 +262,9 @@ std::pair<int, std::string> ModelManager::http_get(int port, const std::string &
264
  beast::tcp_stream stream(ioc);
265
  auto const results = resolver.resolve(worker_host_, std::to_string(port));
266
  stream.connect(results);
 
 
 
267
 
268
  http::request<http::string_body> req{http::verb::get, target, 11};
269
  req.set(http::field::host, worker_host_);
 
172
  json out;
173
  out["status"] = (active_ && is_alive(active_->pid)) ? "ready" : "no_active_model";
174
  out["switch_in_progress"] = switch_in_progress_;
175
+ // Process id and port are internal details; keep them out of the
176
+ // unauthenticated status view.
177
  if (active_ && is_alive(active_->pid)) {
178
  out["current_model"] = active_->model;
179
  out["last_loaded"] = active_->last_loaded;
 
 
180
  } else {
181
  out["current_model"] = nullptr;
182
  out["last_loaded"] = nullptr;
 
 
183
  }
184
  return out;
185
  }
 
262
  beast::tcp_stream stream(ioc);
263
  auto const results = resolver.resolve(worker_host_, std::to_string(port));
264
  stream.connect(results);
265
+ // Readiness probe: a worker that accepts but never answers must not stall
266
+ // the switch forever.
267
+ set_socket_timeouts(stream.socket().native_handle(), 5, 5);
268
 
269
  http::request<http::string_body> req{http::verb::get, target, 11};
270
  req.set(http::field::host, worker_host_);
cpp/request_parsing.cpp CHANGED
@@ -1,11 +1,12 @@
1
  #include "request_parsing.h"
2
 
3
  #include <algorithm>
 
4
  #include <sstream>
5
 
6
- static int estimate_text_tokens_rough(const std::string &text) {
7
  if (text.empty()) return 0;
8
- return std::max(1, static_cast<int>((text.size() + 3) / 4));
9
  }
10
 
11
  static std::string flatten_json_content(const json &content) {
@@ -35,6 +36,7 @@ static std::string flatten_json_content(const json &content) {
35
  std::optional<TokenEstimate> estimate_chat_tokens(
36
  const json &payload,
37
  const LimitsConfig &limits,
 
38
  std::string &error) {
39
  if (!payload.is_object()) {
40
  error = "Expected JSON object";
@@ -45,38 +47,60 @@ std::optional<TokenEstimate> estimate_chat_tokens(
45
  return std::nullopt;
46
  }
47
 
48
- TokenEstimate estimate;
49
- estimate.requested_max_tokens = limits.default_max_tokens;
 
 
50
  if (payload.contains("max_tokens")) {
51
  if (!payload["max_tokens"].is_number_integer()) {
52
  error = "Expected integer 'max_tokens'";
53
  return std::nullopt;
54
  }
55
- estimate.requested_max_tokens = payload["max_tokens"].get<int>();
 
 
 
 
 
 
 
56
  }
57
 
58
- if (estimate.requested_max_tokens <= 0) {
59
  // -1 (and other non-positive values) is a sentinel meaning "unlimited" — use the default
60
- estimate.requested_max_tokens = limits.default_max_tokens;
 
 
 
 
61
  }
62
 
 
63
  for (const auto &message : payload["messages"]) {
64
  if (!message.is_object()) continue;
65
  if (message.contains("role") && message["role"].is_string()) {
66
- estimate.prompt_tokens += estimate_text_tokens_rough(message["role"].get<std::string>());
67
  }
68
  if (message.contains("content")) {
69
- estimate.prompt_tokens += estimate_text_tokens_rough(flatten_json_content(message["content"]));
70
  }
71
- estimate.prompt_tokens += 4;
 
72
  }
 
 
 
73
 
74
- estimate.estimated_total_tokens = estimate.prompt_tokens + estimate.requested_max_tokens;
75
- if (estimate.estimated_total_tokens > limits.max_tokens_per_request) {
76
  error = "Estimated request tokens exceed configured limit";
77
  return std::nullopt;
78
  }
79
 
 
 
 
 
80
  error.clear();
81
  return estimate;
82
  }
 
1
  #include "request_parsing.h"
2
 
3
  #include <algorithm>
4
+ #include <cstdint>
5
  #include <sstream>
6
 
7
+ static int64_t estimate_text_tokens_rough(const std::string &text) {
8
  if (text.empty()) return 0;
9
+ return std::max<int64_t>(1, static_cast<int64_t>((text.size() + 3) / 4));
10
  }
11
 
12
  static std::string flatten_json_content(const json &content) {
 
36
  std::optional<TokenEstimate> estimate_chat_tokens(
37
  const json &payload,
38
  const LimitsConfig &limits,
39
+ size_t body_bytes,
40
  std::string &error) {
41
  if (!payload.is_object()) {
42
  error = "Expected JSON object";
 
47
  return std::nullopt;
48
  }
49
 
50
+ // All arithmetic in 64 bits: an int-sized max_tokens plus a prompt used
51
+ // to wrap negative and sail past every budget check below.
52
+ const int64_t max_per_request = limits.max_tokens_per_request;
53
+ int64_t requested_max_tokens = limits.default_max_tokens;
54
  if (payload.contains("max_tokens")) {
55
  if (!payload["max_tokens"].is_number_integer()) {
56
  error = "Expected integer 'max_tokens'";
57
  return std::nullopt;
58
  }
59
+ // Out-of-range integers (beyond int64) come back saturated by nlohmann
60
+ // as unsigned; treat anything that does not fit as over the limit.
61
+ if (payload["max_tokens"].is_number_unsigned() &&
62
+ payload["max_tokens"].get<uint64_t>() > static_cast<uint64_t>(INT64_MAX)) {
63
+ error = "'max_tokens' exceeds configured limit";
64
+ return std::nullopt;
65
+ }
66
+ requested_max_tokens = payload["max_tokens"].get<int64_t>();
67
  }
68
 
69
+ if (requested_max_tokens <= 0) {
70
  // -1 (and other non-positive values) is a sentinel meaning "unlimited" — use the default
71
+ requested_max_tokens = limits.default_max_tokens;
72
+ }
73
+ if (requested_max_tokens > max_per_request) {
74
+ error = "'max_tokens' exceeds configured limit";
75
+ return std::nullopt;
76
  }
77
 
78
+ int64_t prompt_tokens = 0;
79
  for (const auto &message : payload["messages"]) {
80
  if (!message.is_object()) continue;
81
  if (message.contains("role") && message["role"].is_string()) {
82
+ prompt_tokens += estimate_text_tokens_rough(message["role"].get<std::string>());
83
  }
84
  if (message.contains("content")) {
85
+ prompt_tokens += estimate_text_tokens_rough(flatten_json_content(message["content"]));
86
  }
87
+ prompt_tokens += 4;
88
+ if (prompt_tokens > max_per_request) break; // already over; stop walking
89
  }
90
+ // Floor the estimate on the raw body size so payload that the text walker
91
+ // does not see (tools, image parts, unknown keys) is still budgeted.
92
+ prompt_tokens = std::max(prompt_tokens, static_cast<int64_t>(body_bytes / 4));
93
 
94
+ const int64_t total = prompt_tokens + requested_max_tokens;
95
+ if (prompt_tokens > max_per_request || total > max_per_request) {
96
  error = "Estimated request tokens exceed configured limit";
97
  return std::nullopt;
98
  }
99
 
100
+ TokenEstimate estimate;
101
+ estimate.prompt_tokens = static_cast<int>(prompt_tokens);
102
+ estimate.requested_max_tokens = static_cast<int>(requested_max_tokens);
103
+ estimate.estimated_total_tokens = static_cast<int>(total);
104
  error.clear();
105
  return estimate;
106
  }
cpp/request_parsing.h CHANGED
@@ -5,9 +5,13 @@
5
 
6
  #include "llm_manager_types.h"
7
 
 
 
 
8
  std::optional<TokenEstimate> estimate_chat_tokens(
9
  const json &payload,
10
  const LimitsConfig &limits,
 
11
  std::string &error);
12
 
13
  bool request_stream_enabled(const json &payload);
 
5
 
6
  #include "llm_manager_types.h"
7
 
8
+ // `body_bytes` is the size of the raw request body: it is used as a floor for
9
+ // the prompt estimate so fields the text walker does not see (tools, image
10
+ // parts, unknown keys) still count against the budget.
11
  std::optional<TokenEstimate> estimate_chat_tokens(
12
  const json &payload,
13
  const LimitsConfig &limits,
14
+ size_t body_bytes,
15
  std::string &error);
16
 
17
  bool request_stream_enabled(const json &payload);
cpp/runtime_components.cpp CHANGED
@@ -3,6 +3,7 @@
3
  #include "config.h"
4
  #include "http_helpers.h"
5
  #include "model_manager.h"
 
6
 
7
  #include <algorithm>
8
  #include <cctype>
@@ -124,6 +125,10 @@ std::shared_ptr<RequestContext> RequestRegistry::cancel_request(
124
  if (previous_state) *previous_state = expected;
125
  if (expected == RequestState::RUNNING) {
126
  ctx->state.store(RequestState::CANCELLED);
 
 
 
 
127
  }
128
  }
129
  return ctx;
@@ -145,6 +150,7 @@ std::vector<std::shared_ptr<RequestContext>> RequestRegistry::cancel_all() {
145
  ctx->cv.notify_all();
146
  } else if (expected == RequestState::RUNNING) {
147
  ctx->state.store(RequestState::CANCELLED);
 
148
  }
149
  out.push_back(ctx);
150
  }
@@ -358,7 +364,7 @@ void Scheduler::worker_loop() {
358
  }
359
 
360
  try {
361
- auto [status, body] = forward_chat(*worker, ctx->request_body);
362
  if (ctx->cancelled.load()) {
363
  registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
364
  continue;
@@ -368,39 +374,66 @@ void Scheduler::worker_loop() {
368
  }
369
  registry_.complete(ctx, RequestState::DONE, {status, body});
370
  } catch (const std::exception &e) {
 
 
 
 
 
 
371
  log_line("request_id=" + ctx->request_id + " scheduler_exception=" + std::string(e.what()));
372
- registry_.complete(ctx, RequestState::FAILED, {500, json({{"error", e.what()}}).dump()});
373
  } catch (...) {
 
 
 
 
374
  log_line("request_id=" + ctx->request_id + " scheduler_exception=unknown");
375
- registry_.complete(ctx, RequestState::FAILED, {500, json({{"error", "Unknown exception"}}).dump()});
376
  }
377
  }
378
  }
379
 
380
  ApiKeyAuth::ApiKeyAuth(const ManagerConfig &config)
381
- : header_name_(config.auth.header), scheme_(config.auth.scheme) {
 
 
382
  for (const auto &record : config.api_keys) {
383
- records_by_secret_.emplace(record.secret, record);
 
 
 
 
384
  }
385
  }
386
 
387
  bool ApiKeyAuth::enabled() const {
388
- return !records_by_secret_.empty();
389
  }
390
 
391
  std::optional<ApiKeyRecord> ApiKeyAuth::authenticate(
392
  const http::request<http::string_body> &req,
393
  std::string &error) const {
394
- if (!enabled()) {
 
 
 
 
395
  error.clear();
396
- return ApiKeyRecord{"anonymous", "", Role::ADMIN, true};
 
 
 
 
397
  }
398
 
399
  const auto token = extract_bearer_token(req, error);
400
  if (!token) return std::nullopt;
401
 
402
- const auto it = records_by_secret_.find(*token);
403
- if (it == records_by_secret_.end()) {
 
 
 
404
  error = "Invalid API key";
405
  return std::nullopt;
406
  }
 
3
  #include "config.h"
4
  #include "http_helpers.h"
5
  #include "model_manager.h"
6
+ #include "sha256.h"
7
 
8
  #include <algorithm>
9
  #include <cctype>
 
125
  if (previous_state) *previous_state = expected;
126
  if (expected == RequestState::RUNNING) {
127
  ctx->state.store(RequestState::CANCELLED);
128
+ // Soft cancel: drop the upstream connection for this request only.
129
+ // llama-server abandons the task when its client disconnects, so
130
+ // no worker restart is needed and other users are unaffected.
131
+ ctx->trigger_abort();
132
  }
133
  }
134
  return ctx;
 
150
  ctx->cv.notify_all();
151
  } else if (expected == RequestState::RUNNING) {
152
  ctx->state.store(RequestState::CANCELLED);
153
+ ctx->trigger_abort();
154
  }
155
  out.push_back(ctx);
156
  }
 
364
  }
365
 
366
  try {
367
+ auto [status, body] = forward_chat(*worker, ctx);
368
  if (ctx->cancelled.load()) {
369
  registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
370
  continue;
 
374
  }
375
  registry_.complete(ctx, RequestState::DONE, {status, body});
376
  } catch (const std::exception &e) {
377
+ if (ctx->cancelled.load()) {
378
+ // Expected: the cancel path shut the upstream socket under us.
379
+ registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
380
+ continue;
381
+ }
382
+ // Upstream details stay in the log; clients get a generic error.
383
  log_line("request_id=" + ctx->request_id + " scheduler_exception=" + std::string(e.what()));
384
+ registry_.complete(ctx, RequestState::FAILED, {502, R"({"error":"Upstream inference error"})"});
385
  } catch (...) {
386
+ if (ctx->cancelled.load()) {
387
+ registry_.complete(ctx, RequestState::CANCELLED, {499, R"({"error":"Request cancelled"})"});
388
+ continue;
389
+ }
390
  log_line("request_id=" + ctx->request_id + " scheduler_exception=unknown");
391
+ registry_.complete(ctx, RequestState::FAILED, {502, R"({"error":"Upstream inference error"})"});
392
  }
393
  }
394
  }
395
 
396
  ApiKeyAuth::ApiKeyAuth(const ManagerConfig &config)
397
+ : header_name_(config.auth.header),
398
+ scheme_(config.auth.scheme),
399
+ allow_anonymous_(config.auth.allow_anonymous) {
400
  for (const auto &record : config.api_keys) {
401
+ // Disabled records are kept so a presented-but-disabled key reports
402
+ // "disabled" instead of "invalid"; only enabled ones count as usable.
403
+ if (record.secret_sha256.empty()) continue;
404
+ records_by_hash_.emplace(record.secret_sha256, record);
405
+ if (record.enabled) ++usable_keys_;
406
  }
407
  }
408
 
409
  bool ApiKeyAuth::enabled() const {
410
+ return usable_keys_ > 0;
411
  }
412
 
413
  std::optional<ApiKeyRecord> ApiKeyAuth::authenticate(
414
  const http::request<http::string_body> &req,
415
  std::string &error) const {
416
+ const bool has_header = req.find(header_name_) != req.end();
417
+
418
+ // Fail closed: with no usable keys and no explicit anonymous opt-in,
419
+ // nothing is let through. Anonymous access is never more than USER.
420
+ if (!has_header && allow_anonymous_) {
421
  error.clear();
422
+ return ApiKeyRecord{"anonymous", "", "", Role::USER, true};
423
+ }
424
+ if (!enabled() && !has_header) {
425
+ error = "Authentication not configured";
426
+ return std::nullopt;
427
  }
428
 
429
  const auto token = extract_bearer_token(req, error);
430
  if (!token) return std::nullopt;
431
 
432
+ // Hash first so lookup time does not depend on the secret's bytes, then
433
+ // confirm with a constant-time compare of the digests.
434
+ const std::string digest = sha256_hex(*token);
435
+ const auto it = records_by_hash_.find(digest);
436
+ if (it == records_by_hash_.end() || !constant_time_equals(it->first, digest)) {
437
  error = "Invalid API key";
438
  return std::nullopt;
439
  }
cpp/runtime_components.h CHANGED
@@ -157,7 +157,10 @@ public:
157
  private:
158
  std::string header_name_;
159
  std::string scheme_;
160
- std::unordered_map<std::string, ApiKeyRecord> records_by_secret_;
 
 
 
161
 
162
  std::optional<std::string> extract_bearer_token(
163
  const http::request<http::string_body> &req,
 
157
  private:
158
  std::string header_name_;
159
  std::string scheme_;
160
+ bool allow_anonymous_ = false;
161
+ size_t usable_keys_ = 0;
162
+ // Keyed by lowercase hex SHA-256 of the secret.
163
+ std::unordered_map<std::string, ApiKeyRecord> records_by_hash_;
164
 
165
  std::optional<std::string> extract_bearer_token(
166
  const http::request<http::string_body> &req,
cpp/server.cpp CHANGED
@@ -2,6 +2,7 @@
2
 
3
  #include <boost/beast/core.hpp>
4
 
 
5
  #include "http_helpers.h"
6
  #include "model_manager.h"
7
  #include "request_parsing.h"
@@ -9,17 +10,14 @@
9
 
10
  #include <algorithm>
11
  #include <atomic>
 
12
  #include <sstream>
 
13
  #include <utility>
14
 
15
- #include <sys/socket.h>
16
- #include <sys/time.h>
17
-
18
  namespace beast = boost::beast;
19
  namespace http = beast::http;
20
 
21
- static std::atomic<uint64_t> g_req_id{1};
22
-
23
  static std::string build_sse_event(const json &payload) {
24
  return "data: " + payload.dump() + "\n\n";
25
  }
@@ -231,21 +229,60 @@ static std::string build_buffered_stream_response(const std::string &completion_
231
  return oss.str();
232
  }
233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
234
  http::response<http::string_body> handle_request(
235
- ModelManager &manager,
236
- const ManagerConfig &config,
237
- const ApiKeyAuth &auth,
238
- RateLimiterStore &rate_limiter,
239
- RequestRegistry &registry,
240
- MetricsRegistry &metrics,
241
- Scheduler &scheduler,
242
  http::request<http::string_body> &&req) {
 
 
 
 
 
 
 
 
243
  const auto start = std::chrono::steady_clock::now();
244
- const auto req_id_num = g_req_id.fetch_add(1);
245
- const std::string request_id = std::to_string(req_id_num);
246
  const std::string target = req.target().to_string();
247
  const std::string method = req.method_string().to_string();
248
- const std::string path = target.substr(0, target.find('?'));
 
 
249
  auto authenticated = std::optional<ApiKeyRecord>{};
250
 
251
  metrics.inc_requests_total();
@@ -255,8 +292,14 @@ http::response<http::string_body> handle_request(
255
  ~InflightGuard() { metrics.dec_requests_inflight(); }
256
  } inflight_guard{metrics};
257
 
258
- log_line("request_id=" + request_id + " method=" + method + " path=" + target);
259
- if (!req.body().empty()) {
 
 
 
 
 
 
260
  log_line("request_id=" + request_id + " body=" + truncate_body(req.body()));
261
  }
262
 
@@ -304,7 +347,9 @@ http::response<http::string_body> handle_request(
304
  std::string auth_error;
305
  authenticated = auth.authenticate(req, auth_error);
306
  if (!authenticated) {
307
- return json_response(http::status::unauthorized, {{"error", auth_error}});
 
 
308
  }
309
  if (minimum_role == Role::ADMIN && authenticated->role != Role::ADMIN) {
310
  return json_response(http::status::forbidden, {{"error", "Admin role required"}});
@@ -316,6 +361,24 @@ http::response<http::string_body> handle_request(
316
  };
317
 
318
  try {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
319
  if (path == "/health" && req.method() == http::verb::get) {
320
  return json_response(http::status::ok, manager.models_view());
321
  }
@@ -352,6 +415,15 @@ http::response<http::string_body> handle_request(
352
  if (model.empty()) {
353
  return json_response(http::status::bad_request, {{"error", "Expected string 'model' or 'model_name'"}});
354
  }
 
 
 
 
 
 
 
 
 
355
 
356
  std::string err;
357
  bool ok = manager.switch_model(model, err);
@@ -389,30 +461,26 @@ http::response<http::string_body> handle_request(
389
  if (auto cancel_id = extract_cancel_request_id(path)) {
390
  if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
391
  auto ctx = registry.find(*cancel_id);
392
- if (!ctx) {
 
 
393
  return json_response(http::status::not_found, {{"error", "Unknown request id"}});
394
  }
395
- if (authenticated->role != Role::ADMIN && authenticated->key_id != ctx->api_key_id) {
396
- return json_response(http::status::forbidden, {{"error", "Cannot cancel another API key request"}});
397
- }
398
 
 
 
 
 
399
  RequestState previous_state = RequestState::QUEUED;
400
  registry.cancel_request(*cancel_id, &previous_state);
401
  const bool did_cancel = previous_state == RequestState::QUEUED ||
402
  previous_state == RequestState::RUNNING;
403
  if (did_cancel) metrics.add_cancellations_total();
404
- std::string restart_error;
405
- bool restarted = true;
406
- if (previous_state == RequestState::RUNNING) {
407
- restarted = manager.restart_active(restart_error);
408
- if (restarted) metrics.inc_worker_restarts_total();
409
- }
410
 
411
  json payload = {
412
  {"cancelled_request_id", *cancel_id},
413
  {"state", state_to_string(ctx->state.load())}
414
  };
415
- if (!restarted) payload["restart_error"] = restart_error;
416
  return json_response(http::status::ok, payload);
417
  }
418
  }
@@ -438,7 +506,7 @@ http::response<http::string_body> handle_request(
438
  }
439
 
440
  std::string token_error;
441
- auto estimate = estimate_chat_tokens(payload, config.limits, token_error);
442
  if (!estimate) {
443
  return json_response(http::status::bad_request, {{"error", token_error}});
444
  }
@@ -500,20 +568,15 @@ http::response<http::string_body> handle_request(
500
  }
501
  if (!finished) {
502
  lock.unlock();
 
 
503
  RequestState timed_out_state = RequestState::QUEUED;
504
  registry.cancel_request(request_id, &timed_out_state);
505
  metrics.add_cancellations_total();
506
- std::string restart_error;
507
- bool restarted = true;
508
- if (timed_out_state == RequestState::RUNNING) {
509
- restarted = manager.restart_active(restart_error);
510
- if (restarted) metrics.inc_worker_restarts_total();
511
- }
512
  json timeout_payload = {
513
  {"error", "Request timed out"},
514
  {"state", state_to_string(ctx->state.load())}
515
  };
516
- if (!restarted) timeout_payload["restart_error"] = restart_error;
517
  return json_response(http::status::gateway_timeout, timeout_payload);
518
  }
519
 
@@ -558,17 +621,28 @@ http::response<http::string_body> handle_request(
558
  }
559
 
560
  if (req.method() == http::verb::get) {
561
- // /slots can expose other users' prompt content and /metrics exposes
562
- // server internals; don't proxy them anonymously. Static UI assets
563
- // and /props stay open so the browser UI can load.
564
- if (path == "/slots" || path == "/metrics") {
 
 
 
 
 
 
 
 
 
 
 
565
  if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
566
  }
567
  auto worker = manager.active_worker();
568
  if (!worker) {
569
  return json_response(http::status::service_unavailable, {{"error", "No active model"}});
570
  }
571
- auto upstream = forward_get_to_worker(*worker, target);
572
  http::response<http::string_body> res{
573
  static_cast<http::status>(upstream.status), req.version()};
574
  res.set(http::field::content_type, upstream.content_type);
@@ -595,38 +669,60 @@ http::response<http::string_body> handle_request(
595
 
596
  return json_response(http::status::not_found, {{"error", "Not found"}});
597
  } catch (const std::exception &e) {
 
 
598
  log_line("request_id=" + request_id + " exception=" + std::string(e.what()));
599
- return json_response(http::status::internal_server_error, {{"error", e.what()}});
600
  } catch (...) {
601
  log_line("request_id=" + request_id + " exception=unknown");
602
- return json_response(http::status::internal_server_error, {{"error", "Unknown exception"}});
603
  }
604
  }
605
 
606
- void do_session(
607
- boost::asio::ip::tcp::socket socket,
608
- ModelManager &manager,
609
- const ManagerConfig &config,
610
- const ApiKeyAuth &auth,
611
- RateLimiterStore &rate_limiter,
612
- RequestRegistry &registry,
613
- MetricsRegistry &metrics,
614
- Scheduler &scheduler) {
615
  // Receive timeout: each connection pins a detached thread, so idle
616
  // keep-alive connections and stalled senders must eventually release it.
 
 
 
617
  {
618
- timeval tv{};
619
- tv.tv_sec = 60;
620
- setsockopt(socket.native_handle(), SOL_SOCKET, SO_RCVTIMEO,
621
- reinterpret_cast<const char *>(&tv), sizeof(tv));
622
  }
 
623
  try {
624
  beast::flat_buffer buffer;
625
  for (;;) {
626
- http::request<http::string_body> req;
627
- http::read(socket, buffer, req);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
628
  const bool keep_alive = req.keep_alive();
629
- auto res = handle_request(manager, config, auth, rate_limiter, registry, metrics, scheduler, std::move(req));
630
  http::write(socket, res);
631
  if (!keep_alive || res.need_eof()) break;
632
  }
 
2
 
3
  #include <boost/beast/core.hpp>
4
 
5
+ #include "config.h"
6
  #include "http_helpers.h"
7
  #include "model_manager.h"
8
  #include "request_parsing.h"
 
10
 
11
  #include <algorithm>
12
  #include <atomic>
13
+ #include <regex>
14
  #include <sstream>
15
+ #include <unordered_set>
16
  #include <utility>
17
 
 
 
 
18
  namespace beast = boost::beast;
19
  namespace http = beast::http;
20
 
 
 
21
  static std::string build_sse_event(const json &payload) {
22
  return "data: " + payload.dump() + "\n\n";
23
  }
 
229
  return oss.str();
230
  }
231
 
232
+ // Model identifiers go straight onto llama-server's command line as the -hf
233
+ // value, so only accept the owner/repo[:quant] shape and nothing that looks
234
+ // like a flag, a path, or contains control characters.
235
+ static bool is_valid_model_identifier(const std::string &model) {
236
+ if (model.empty() || model.size() > 256) return false;
237
+ static const std::regex pattern(
238
+ R"(^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*(:[A-Za-z0-9][A-Za-z0-9_.-]*)?$)");
239
+ if (!std::regex_match(model, pattern)) return false;
240
+ return model.find("..") == std::string::npos;
241
+ }
242
+
243
+ static bool is_allowed_model(const ModelsConfig &models, const std::string &model) {
244
+ return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
245
+ }
246
+
247
+ // Resolves the client address used for per-source rate limiting. The first
248
+ // X-Forwarded-For hop is used only when the deployment says the proxy in
249
+ // front of us is trusted (HF Spaces); otherwise the TCP peer.
250
+ static std::string client_address(
251
+ const http::request<http::string_body> &req,
252
+ const std::string &peer_address,
253
+ bool trust_proxy_headers) {
254
+ if (trust_proxy_headers) {
255
+ const auto it = req.find("X-Forwarded-For");
256
+ if (it != req.end()) {
257
+ std::string value = it->value().to_string();
258
+ const auto comma = value.find(',');
259
+ if (comma != std::string::npos) value = value.substr(0, comma);
260
+ value = trim_copy(value);
261
+ if (!value.empty() && value.size() <= 64) return value;
262
+ }
263
+ }
264
+ return peer_address;
265
+ }
266
+
267
  http::response<http::string_body> handle_request(
268
+ ServerDeps &deps,
269
+ const std::string &peer_address,
 
 
 
 
 
270
  http::request<http::string_body> &&req) {
271
+ ModelManager &manager = deps.manager;
272
+ const ManagerConfig &config = deps.config;
273
+ const ApiKeyAuth &auth = deps.auth;
274
+ RateLimiterStore &rate_limiter = deps.rate_limiter;
275
+ RequestRegistry &registry = deps.registry;
276
+ MetricsRegistry &metrics = deps.metrics;
277
+ Scheduler &scheduler = deps.scheduler;
278
+
279
  const auto start = std::chrono::steady_clock::now();
280
+ const std::string request_id = generate_request_id();
 
281
  const std::string target = req.target().to_string();
282
  const std::string method = req.method_string().to_string();
283
+ const auto query_pos = target.find('?');
284
+ const std::string raw_path = target.substr(0, query_pos);
285
+ const std::string query = query_pos == std::string::npos ? "" : target.substr(query_pos);
286
  auto authenticated = std::optional<ApiKeyRecord>{};
287
 
288
  metrics.inc_requests_total();
 
292
  ~InflightGuard() { metrics.dec_requests_inflight(); }
293
  } inflight_guard{metrics};
294
 
295
+ // Canonical path for routing. Anything that does not normalise (escapes
296
+ // the root, control chars, bad percent-encoding) is simply not found.
297
+ const auto normalized = normalize_path(raw_path);
298
+ const std::string path = normalized ? *normalized : std::string();
299
+
300
+ log_line("request_id=" + request_id + " method=" + method + " path=" + raw_path +
301
+ " body_bytes=" + std::to_string(req.body().size()));
302
+ if (config.logging.log_request_bodies && !req.body().empty()) {
303
  log_line("request_id=" + request_id + " body=" + truncate_body(req.body()));
304
  }
305
 
 
347
  std::string auth_error;
348
  authenticated = auth.authenticate(req, auth_error);
349
  if (!authenticated) {
350
+ auto res = json_response(http::status::unauthorized, {{"error", auth_error}});
351
+ res.set(http::field::www_authenticate, config.auth.scheme);
352
+ return res;
353
  }
354
  if (minimum_role == Role::ADMIN && authenticated->role != Role::ADMIN) {
355
  return json_response(http::status::forbidden, {{"error", "Admin role required"}});
 
361
  };
362
 
363
  try {
364
+ // Pre-auth per-source limit: protects the thread pool and the auth
365
+ // path itself from a single abusive client.
366
+ if (config.rate_limit.per_ip_requests_per_minute > 0) {
367
+ const std::string source = client_address(req, peer_address, config.rate_limit.trust_proxy_headers);
368
+ auto decision = deps.ip_rate_limiter.allow("ip:" + source, 0);
369
+ if (!decision.allowed) {
370
+ metrics.inc_rate_limited_total();
371
+ return json_response_with_retry_after(
372
+ http::status::too_many_requests,
373
+ {{"error", "Rate limit exceeded: requests from this address"}},
374
+ decision.retry_after_sec);
375
+ }
376
+ }
377
+
378
+ if (!normalized) {
379
+ return json_response(http::status::not_found, {{"error", "Not found"}});
380
+ }
381
+
382
  if (path == "/health" && req.method() == http::verb::get) {
383
  return json_response(http::status::ok, manager.models_view());
384
  }
 
415
  if (model.empty()) {
416
  return json_response(http::status::bad_request, {{"error", "Expected string 'model' or 'model_name'"}});
417
  }
418
+ if (!is_valid_model_identifier(model)) {
419
+ return json_response(http::status::bad_request,
420
+ {{"error", "Invalid model identifier; expected owner/repo[:quant]"}});
421
+ }
422
+ if (!is_allowed_model(config.models, model)) {
423
+ return json_response(http::status::bad_request,
424
+ {{"error", "Model is not in the configured allowlist"},
425
+ {"allowed_models", config.models.allowed}});
426
+ }
427
 
428
  std::string err;
429
  bool ok = manager.switch_model(model, err);
 
461
  if (auto cancel_id = extract_cancel_request_id(path)) {
462
  if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
463
  auto ctx = registry.find(*cancel_id);
464
+ // Same answer for "unknown" and "not yours" so request ids of
465
+ // other keys cannot be probed.
466
+ if (!ctx || (authenticated->role != Role::ADMIN && authenticated->key_id != ctx->api_key_id)) {
467
  return json_response(http::status::not_found, {{"error", "Unknown request id"}});
468
  }
 
 
 
469
 
470
+ // Soft cancel only: a queued request is dropped, a running one
471
+ // has its upstream connection closed. The worker is never
472
+ // restarted from here (that is the admin-only /stop), so one
473
+ // key cannot take the service down for everyone else.
474
  RequestState previous_state = RequestState::QUEUED;
475
  registry.cancel_request(*cancel_id, &previous_state);
476
  const bool did_cancel = previous_state == RequestState::QUEUED ||
477
  previous_state == RequestState::RUNNING;
478
  if (did_cancel) metrics.add_cancellations_total();
 
 
 
 
 
 
479
 
480
  json payload = {
481
  {"cancelled_request_id", *cancel_id},
482
  {"state", state_to_string(ctx->state.load())}
483
  };
 
484
  return json_response(http::status::ok, payload);
485
  }
486
  }
 
506
  }
507
 
508
  std::string token_error;
509
+ auto estimate = estimate_chat_tokens(payload, config.limits, req.body().size(), token_error);
510
  if (!estimate) {
511
  return json_response(http::status::bad_request, {{"error", token_error}});
512
  }
 
568
  }
569
  if (!finished) {
570
  lock.unlock();
571
+ // Soft cancel: closes this request's upstream connection; the
572
+ // worker keeps serving everyone else.
573
  RequestState timed_out_state = RequestState::QUEUED;
574
  registry.cancel_request(request_id, &timed_out_state);
575
  metrics.add_cancellations_total();
 
 
 
 
 
 
576
  json timeout_payload = {
577
  {"error", "Request timed out"},
578
  {"state", state_to_string(ctx->state.load())}
579
  };
 
580
  return json_response(http::status::gateway_timeout, timeout_payload);
581
  }
582
 
 
621
  }
622
 
623
  if (req.method() == http::verb::get) {
624
+ // Allowlist, not blocklist: only the paths the bundled web UI needs
625
+ // are proxied, matched on the canonical (decoded) path so encoded
626
+ // variants cannot slip past. /props exposes model and template
627
+ // settings, /slots other users' prompts, /metrics server internals
628
+ // — those need a key.
629
+ static const std::unordered_set<std::string> kPublicPaths = {
630
+ "/", "/index.html", "/index.html.gz", "/favicon.ico"};
631
+ static const std::unordered_set<std::string> kAuthenticatedPaths = {
632
+ "/props", "/v1/models", "/slots", "/metrics"};
633
+ const bool is_public = kPublicPaths.count(path) > 0;
634
+ const bool needs_auth = kAuthenticatedPaths.count(path) > 0;
635
+ if (!is_public && !needs_auth) {
636
+ return json_response(http::status::not_found, {{"error", "Not found"}});
637
+ }
638
+ if (needs_auth) {
639
  if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
640
  }
641
  auto worker = manager.active_worker();
642
  if (!worker) {
643
  return json_response(http::status::service_unavailable, {{"error", "No active model"}});
644
  }
645
+ auto upstream = forward_get_to_worker(*worker, path + query);
646
  http::response<http::string_body> res{
647
  static_cast<http::status>(upstream.status), req.version()};
648
  res.set(http::field::content_type, upstream.content_type);
 
669
 
670
  return json_response(http::status::not_found, {{"error", "Not found"}});
671
  } catch (const std::exception &e) {
672
+ // Exception text can reveal internals (paths, library messages); log
673
+ // it, return a generic message.
674
  log_line("request_id=" + request_id + " exception=" + std::string(e.what()));
675
+ return json_response(http::status::internal_server_error, {{"error", "Internal server error"}});
676
  } catch (...) {
677
  log_line("request_id=" + request_id + " exception=unknown");
678
+ return json_response(http::status::internal_server_error, {{"error", "Internal server error"}});
679
  }
680
  }
681
 
682
+ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
683
+ struct SessionGuard {
684
+ std::atomic<int> &counter;
685
+ ~SessionGuard() { counter.fetch_sub(1); }
686
+ } session_guard{deps.active_sessions};
687
+
688
+ const ManagerConfig &config = deps.config;
 
 
689
  // Receive timeout: each connection pins a detached thread, so idle
690
  // keep-alive connections and stalled senders must eventually release it.
691
+ set_socket_timeouts(socket.native_handle(), config.server.read_timeout_sec, 60);
692
+
693
+ std::string peer_address;
694
  {
695
+ boost::system::error_code ec;
696
+ const auto endpoint = socket.remote_endpoint(ec);
697
+ if (!ec) peer_address = endpoint.address().to_string();
 
698
  }
699
+
700
  try {
701
  beast::flat_buffer buffer;
702
  for (;;) {
703
+ // Explicit limits instead of Beast defaults: oversized headers or
704
+ // bodies are rejected before they are buffered in full.
705
+ http::request_parser<http::string_body> parser;
706
+ parser.header_limit(16 * 1024);
707
+ parser.body_limit(static_cast<uint64_t>(config.limits.max_request_body_bytes));
708
+ beast::error_code read_ec;
709
+ http::read(socket, buffer, parser, read_ec);
710
+ if (read_ec == http::error::body_limit || read_ec == http::error::header_limit) {
711
+ http::response<http::string_body> res{http::status::payload_too_large, 11};
712
+ res.set(http::field::content_type, "application/json");
713
+ res.set(http::field::server, "llm-manager");
714
+ res.keep_alive(false);
715
+ res.body() = R"({"error":"Request too large"})";
716
+ res.prepare_payload();
717
+ beast::error_code write_ec;
718
+ http::write(socket, res, write_ec);
719
+ break;
720
+ }
721
+ if (read_ec) throw beast::system_error{read_ec};
722
+
723
+ http::request<http::string_body> req = parser.release();
724
  const bool keep_alive = req.keep_alive();
725
+ auto res = handle_request(deps, peer_address, std::move(req));
726
  http::write(socket, res);
727
  if (!keep_alive || res.need_eof()) break;
728
  }
cpp/server.h CHANGED
@@ -3,6 +3,8 @@
3
  #include <boost/asio.hpp>
4
  #include <boost/beast/http.hpp>
5
 
 
 
6
  #include "llm_manager_types.h"
7
 
8
  class ModelManager;
@@ -14,22 +16,26 @@ class Scheduler;
14
 
15
  namespace http = boost::beast::http;
16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  http::response<http::string_body> handle_request(
18
- ModelManager &manager,
19
- const ManagerConfig &config,
20
- const ApiKeyAuth &auth,
21
- RateLimiterStore &rate_limiter,
22
- RequestRegistry &registry,
23
- MetricsRegistry &metrics,
24
- Scheduler &scheduler,
25
  http::request<http::string_body> &&req);
26
 
27
- void do_session(
28
- boost::asio::ip::tcp::socket socket,
29
- ModelManager &manager,
30
- const ManagerConfig &config,
31
- const ApiKeyAuth &auth,
32
- RateLimiterStore &rate_limiter,
33
- RequestRegistry &registry,
34
- MetricsRegistry &metrics,
35
- Scheduler &scheduler);
 
3
  #include <boost/asio.hpp>
4
  #include <boost/beast/http.hpp>
5
 
6
+ #include <atomic>
7
+
8
  #include "llm_manager_types.h"
9
 
10
  class ModelManager;
 
16
 
17
  namespace http = boost::beast::http;
18
 
19
+ // Everything a session needs; owned by main() for the process lifetime.
20
+ struct ServerDeps {
21
+ ModelManager &manager;
22
+ const ManagerConfig &config;
23
+ const ApiKeyAuth &auth;
24
+ RateLimiterStore &rate_limiter;
25
+ // Pre-auth per-source limiter (keyed by client address).
26
+ RateLimiterStore &ip_rate_limiter;
27
+ RequestRegistry &registry;
28
+ MetricsRegistry &metrics;
29
+ Scheduler &scheduler;
30
+ // Open sessions, for the connection cap.
31
+ std::atomic<int> &active_sessions;
32
+ };
33
+
34
  http::response<http::string_body> handle_request(
35
+ ServerDeps &deps,
36
+ const std::string &peer_address,
 
 
 
 
 
37
  http::request<http::string_body> &&req);
38
 
39
+ // Takes ownership of one accepted connection; decrements
40
+ // deps.active_sessions on exit.
41
+ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps);
 
 
 
 
 
 
cpp/sha256.h ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma once
2
+
3
+ // Minimal, dependency-free SHA-256 used to store and compare API key secrets
4
+ // as digests. Not performance critical: one hash per authenticated request.
5
+
6
+ #include <array>
7
+ #include <cstdint>
8
+ #include <cstring>
9
+ #include <string>
10
+
11
+ namespace sha256_detail {
12
+
13
+ inline uint32_t rotr(uint32_t x, uint32_t n) { return (x >> n) | (x << (32 - n)); }
14
+
15
+ constexpr uint32_t K[64] = {
16
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
17
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
18
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
19
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
20
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
21
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
22
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
23
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
24
+
25
+ inline void compress(std::array<uint32_t, 8> &h, const uint8_t block[64]) {
26
+ uint32_t w[64];
27
+ for (int i = 0; i < 16; ++i) {
28
+ w[i] = (uint32_t(block[i * 4]) << 24) | (uint32_t(block[i * 4 + 1]) << 16) |
29
+ (uint32_t(block[i * 4 + 2]) << 8) | uint32_t(block[i * 4 + 3]);
30
+ }
31
+ for (int i = 16; i < 64; ++i) {
32
+ const uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
33
+ const uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
34
+ w[i] = w[i - 16] + s0 + w[i - 7] + s1;
35
+ }
36
+ uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7];
37
+ for (int i = 0; i < 64; ++i) {
38
+ const uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
39
+ const uint32_t ch = (e & f) ^ (~e & g);
40
+ const uint32_t t1 = hh + S1 + ch + K[i] + w[i];
41
+ const uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
42
+ const uint32_t maj = (a & b) ^ (a & c) ^ (b & c);
43
+ const uint32_t t2 = S0 + maj;
44
+ hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2;
45
+ }
46
+ h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += hh;
47
+ }
48
+
49
+ } // namespace sha256_detail
50
+
51
+ // Lowercase hex digest of `input`.
52
+ inline std::string sha256_hex(const std::string &input) {
53
+ std::array<uint32_t, 8> h = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
54
+ 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
55
+ const uint8_t *data = reinterpret_cast<const uint8_t *>(input.data());
56
+ size_t len = input.size();
57
+ while (len >= 64) {
58
+ sha256_detail::compress(h, data);
59
+ data += 64;
60
+ len -= 64;
61
+ }
62
+ uint8_t block[128] = {0};
63
+ std::memcpy(block, data, len);
64
+ block[len] = 0x80;
65
+ const size_t total = (len + 1 + 8 <= 64) ? 64 : 128;
66
+ const uint64_t bit_len = uint64_t(input.size()) * 8;
67
+ for (int i = 0; i < 8; ++i) block[total - 1 - i] = uint8_t(bit_len >> (8 * i));
68
+ sha256_detail::compress(h, block);
69
+ if (total == 128) sha256_detail::compress(h, block + 64);
70
+
71
+ static const char *digits = "0123456789abcdef";
72
+ std::string out;
73
+ out.reserve(64);
74
+ for (uint32_t v : h) {
75
+ for (int shift = 28; shift >= 0; shift -= 4) out.push_back(digits[(v >> shift) & 0xF]);
76
+ }
77
+ return out;
78
+ }
79
+
80
+ // Compares two strings without early exit on the first differing byte.
81
+ inline bool constant_time_equals(const std::string &a, const std::string &b) {
82
+ if (a.size() != b.size()) return false;
83
+ unsigned char diff = 0;
84
+ for (size_t i = 0; i < a.size(); ++i) {
85
+ diff |= static_cast<unsigned char>(a[i]) ^ static_cast<unsigned char>(b[i]);
86
+ }
87
+ return diff == 0;
88
+ }