Dmitry Beresnev commited on
Commit
70dee29
·
1 Parent(s): 6dc320d

fix security volnurabilities

Browse files
Dockerfile CHANGED
@@ -19,14 +19,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
19
  # Clone and build llama.cpp with SSL support for HuggingFace Hub
20
  WORKDIR /build
21
  ARG CACHEBUST=6
22
- # Pinned release tag (supply chain: never build from a moving branch). Bump
23
- # deliberately and re-run the endpoint tests.
24
- ARG LLAMA_CPP_REF=b12367
25
  #ARG BUILD_PROFILE=fast_build
26
  ARG BUILD_PROFILE=fast_runtime
27
  ARG BUILD_JOBS=1
28
- RUN git clone --depth 1 --branch ${LLAMA_CPP_REF} https://github.com/ggerganov/llama.cpp.git && \
29
- cd llama.cpp && \
 
 
 
30
  if [ "${BUILD_PROFILE}" = "fast_runtime" ]; then \
31
  C_FLAGS="-O3 -DNDEBUG"; \
32
  CXX_FLAGS="-O3 -DNDEBUG"; \
@@ -113,6 +116,12 @@ ENV HOME=/home/user \
113
 
114
  EXPOSE 7860
115
 
 
 
 
 
 
 
116
  # Start FastAPI app (which manages llama-server internally)
117
  #CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
118
 
@@ -120,6 +129,7 @@ EXPOSE 7860
120
  # Keep the FastAPI CMD above as the default. Uncomment ONE of the following
121
  # to run the C++ server directly instead of the Python app.
122
  #
 
123
  # Example DeepSeek (4k context):
124
  # CMD ["llama-server", "-hf", "TheBloke/deepseek-llm-7B-chat-GGUF:deepseek-llm-7b-chat.Q4_K_M.gguf",
125
  # "--host", "0.0.0.0", "--port", "7860", "-c", "4096", "-t", "4", "-ngl", "0", "--cont-batching", "-b", "512"]
@@ -150,7 +160,11 @@ EXPOSE 7860
150
  # - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
151
  # (X-Forwarded-For from the HF proxy).
152
  # - MIN_FREE_DISK_MB refuses a model switch when the cache volume is nearly full.
153
- ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
 
 
 
 
154
  PER_IP_REQUESTS_PER_MINUTE=120 \
155
  MANAGER_MAX_SESSIONS=256 \
156
  MAX_REQUEST_BODY_BYTES=1048576 \
@@ -159,7 +173,7 @@ ENV DEFAULT_MODEL=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
159
  MANAGER_HOST=0.0.0.0 \
160
  MANAGER_PORT=7860 \
161
  WORKER_BASE_PORT=8080 \
162
- SWITCH_TIMEOUT_SEC=300 \
163
  REQUEST_TIMEOUT_SEC=300 \
164
  DEFAULT_MAX_TOKENS=2048 \
165
  MAX_TOKENS_PER_REQUEST=4096 \
 
19
  # Clone and build llama.cpp with SSL support for HuggingFace Hub
20
  WORKDIR /build
21
  ARG CACHEBUST=6
22
+ # Pinned to an immutable commit (release tag b12367). Tags can be moved;
23
+ # commits cannot. Bump deliberately and re-run the endpoint tests.
24
+ ARG LLAMA_CPP_REF=fe14cad51bbfb3f1cc8d9cfd9d70e98a123743d3
25
  #ARG BUILD_PROFILE=fast_build
26
  ARG BUILD_PROFILE=fast_runtime
27
  ARG BUILD_JOBS=1
28
+ RUN mkdir llama.cpp && cd llama.cpp && git init -q && \
29
+ git remote add origin https://github.com/ggerganov/llama.cpp.git && \
30
+ git fetch --depth 1 origin ${LLAMA_CPP_REF} && \
31
+ git checkout -q FETCH_HEAD && \
32
+ echo "llama.cpp commit: $(git rev-parse HEAD)" && \
33
  if [ "${BUILD_PROFILE}" = "fast_runtime" ]; then \
34
  C_FLAGS="-O3 -DNDEBUG"; \
35
  CXX_FLAGS="-O3 -DNDEBUG"; \
 
116
 
117
  EXPOSE 7860
118
 
119
+ # The only supported entrypoint is the C++ manager (CMD at the bottom). The
120
+ # legacy FastAPI app is unauthenticated and is not in this image.
121
+ #
122
+ # Historical examples of running llama-server directly (unauthenticated —
123
+ # do not use in production):
124
+
125
  # Start FastAPI app (which manages llama-server internally)
126
  #CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
127
 
 
129
  # Keep the FastAPI CMD above as the default. Uncomment ONE of the following
130
  # to run the C++ server directly instead of the Python app.
131
  #
132
+
133
  # Example DeepSeek (4k context):
134
  # CMD ["llama-server", "-hf", "TheBloke/deepseek-llm-7B-chat-GGUF:deepseek-llm-7b-chat.Q4_K_M.gguf",
135
  # "--host", "0.0.0.0", "--port", "7860", "-c", "4096", "-t", "4", "-ngl", "0", "--cont-batching", "-b", "512"]
 
160
  # - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
161
  # (X-Forwarded-For from the HF proxy).
162
  # - MIN_FREE_DISK_MB refuses a model switch when the cache volume is nearly full.
163
+ # - The allowlist below is the curated set from docs/MODEL_CANDIDATES_FREE_SPACE.md:
164
+ # dense upgrades (Qwen3.5), MoE fast lanes (LFM2.5-8B-A1B, Granite 4.0 H-Tiny,
165
+ # 1-1.5B active), Gemma 4, and one large MoE (Qwen3.6-35B-A3B at IQ3, 12.3 GiB)
166
+ # that needs SWITCH_TIMEOUT_SEC >= 600 on a cold start.
167
+ ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m,unsloth/Qwen3.5-9B-GGUF:Q4_K_M,unsloth/Qwen3.5-4B-GGUF:Q4_K_M,LiquidAI/LFM2.5-8B-A1B-GGUF:Q4_K_M,unsloth/granite-4.0-h-tiny-GGUF:Q4_K_M,unsloth/gemma-4-E4B-it-GGUF:Q4_K_M,unsloth/gemma-4-E2B-it-GGUF:Q4_K_M,unsloth/Qwen3.6-35B-A3B-GGUF:UD-IQ3_XXS \
168
  PER_IP_REQUESTS_PER_MINUTE=120 \
169
  MANAGER_MAX_SESSIONS=256 \
170
  MAX_REQUEST_BODY_BYTES=1048576 \
 
173
  MANAGER_HOST=0.0.0.0 \
174
  MANAGER_PORT=7860 \
175
  WORKER_BASE_PORT=8080 \
176
+ SWITCH_TIMEOUT_SEC=600 \
177
  REQUEST_TIMEOUT_SEC=300 \
178
  DEFAULT_MAX_TOKENS=2048 \
179
  MAX_TOKENS_PER_REQUEST=4096 \
README.md CHANGED
@@ -28,7 +28,8 @@ The legacy Python application is still present under [`python/`](/Users/dmitrybe
28
  ## Security Setup (read before deploying)
29
 
30
  The manager **fails closed**: with no usable API keys configured every authenticated endpoint
31
- (`/v1/chat/completions`, `/switch-model`, `/stop`, cancel, `/props`, `/slots`, `/metrics`) returns 401.
 
32
  Keys are never baked into the image. Configure them as an environment variable / HF Space secret:
33
 
34
  ```bash
@@ -212,7 +213,7 @@ Roles:
212
  - `admin`: can call privileged control endpoints
213
  - `user`: can submit chat requests and cancel owned requests
214
 
215
- If no API keys are configured, the manager currently stays in a compatibility mode and does not enforce auth.
216
 
217
  ## Configuration
218
 
 
28
  ## Security Setup (read before deploying)
29
 
30
  The manager **fails closed**: with no usable API keys configured every authenticated endpoint
31
+ (`/v1/chat/completions`, `/switch-model`, `/stop`, cancel, `/models`, `/queue/metrics`, `/props`, `/slots`,
32
+ `/metrics`) returns 401. Only `/health` (liveness: status and switch flag) and the static web UI are public.
33
  Keys are never baked into the image. Configure them as an environment variable / HF Space secret:
34
 
35
  ```bash
 
213
  - `admin`: can call privileged control endpoints
214
  - `user`: can submit chat requests and cancel owned requests
215
 
216
+ If no usable API keys are configured the manager **fails closed**: every protected endpoint returns 401 (only `/health` and the static web UI stay public). Set `AUTH_ALLOW_ANONYMOUS=true` explicitly to allow unauthenticated callers with the user role.
217
 
218
  ## Configuration
219
 
cpp/config.cpp CHANGED
@@ -255,6 +255,7 @@ static bool is_hex_digest(const std::string &value) {
255
  static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig &auth) {
256
  static const char *kPlaceholders[] = {"change-me", "changeme", "replace", "secret", "password", "test", "example"};
257
  std::unordered_set<std::string> seen_ids;
 
258
  for (auto &record : keys) {
259
  if (!seen_ids.insert(record.key_id).second) {
260
  log_line("config: duplicate api key id '" + record.key_id + "' — disabling later entry");
@@ -289,6 +290,12 @@ static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig
289
  record.enabled = false;
290
  }
291
  }
 
 
 
 
 
 
292
  }
293
  }
294
 
@@ -308,6 +315,8 @@ ManagerConfig load_manager_config() {
308
  cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
309
  cfg.server.max_sessions = std::max(1, get_env_int_or("MANAGER_MAX_SESSIONS", get_toml_int_or(toml, "server", "max_sessions", cfg.server.max_sessions)));
310
  cfg.server.read_timeout_sec = std::max(1, get_env_int_or("MANAGER_READ_TIMEOUT_SEC", get_toml_int_or(toml, "server", "read_timeout_sec", cfg.server.read_timeout_sec)));
 
 
311
 
312
  cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
313
  cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
 
255
  static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig &auth) {
256
  static const char *kPlaceholders[] = {"change-me", "changeme", "replace", "secret", "password", "test", "example"};
257
  std::unordered_set<std::string> seen_ids;
258
+ std::unordered_set<std::string> seen_digests;
259
  for (auto &record : keys) {
260
  if (!seen_ids.insert(record.key_id).second) {
261
  log_line("config: duplicate api key id '" + record.key_id + "' — disabling later entry");
 
290
  record.enabled = false;
291
  }
292
  }
293
+ // The same secret under two ids (possibly two roles) is ambiguous:
294
+ // whichever loaded first would silently win. Fail closed on the rest.
295
+ if (record.enabled && !record.secret_sha256.empty() && !seen_digests.insert(record.secret_sha256).second) {
296
+ log_line("config: api key '" + record.key_id + "' reuses another key's secret — disabled");
297
+ record.enabled = false;
298
+ }
299
  }
300
  }
301
 
 
315
  cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
316
  cfg.server.max_sessions = std::max(1, get_env_int_or("MANAGER_MAX_SESSIONS", get_toml_int_or(toml, "server", "max_sessions", cfg.server.max_sessions)));
317
  cfg.server.read_timeout_sec = std::max(1, get_env_int_or("MANAGER_READ_TIMEOUT_SEC", get_toml_int_or(toml, "server", "read_timeout_sec", cfg.server.read_timeout_sec)));
318
+ cfg.server.header_timeout_sec = std::max(1, get_env_int_or("MANAGER_HEADER_TIMEOUT_SEC", get_toml_int_or(toml, "server", "header_timeout_sec", cfg.server.header_timeout_sec)));
319
+ cfg.server.body_timeout_sec = std::max(cfg.server.header_timeout_sec, get_env_int_or("MANAGER_BODY_TIMEOUT_SEC", get_toml_int_or(toml, "server", "body_timeout_sec", cfg.server.body_timeout_sec)));
320
 
321
  cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
322
  cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
cpp/http_helpers.cpp CHANGED
@@ -9,14 +9,17 @@
9
  #include <ctime>
10
  #include <iomanip>
11
  #include <iostream>
12
- #include <mutex>
13
- #include <random>
14
  #include <sstream>
15
  #include <stdexcept>
16
  #include <vector>
17
 
18
  #include <sys/socket.h>
19
  #include <sys/time.h>
 
 
 
20
 
21
  namespace asio = boost::asio;
22
  namespace beast = boost::beast;
@@ -56,23 +59,38 @@ void set_socket_timeouts(int fd, int recv_sec, int send_sec) {
56
  setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char *>(&snd), sizeof(snd));
57
  }
58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  std::string generate_request_id() {
60
- static std::mutex mu;
61
- static std::mt19937_64 rng = [] {
62
- std::random_device rd;
63
- std::seed_seq seed{rd(), rd(), rd(), rd(), rd(), rd(), rd(), rd()};
64
- return std::mt19937_64(seed);
65
- }();
66
- uint64_t hi, lo;
67
- {
68
- std::lock_guard<std::mutex> lock(mu);
69
- hi = rng();
70
- lo = rng();
71
  }
72
- char buf[33];
73
- std::snprintf(buf, sizeof(buf), "%016llx%016llx",
74
- static_cast<unsigned long long>(hi), static_cast<unsigned long long>(lo));
75
- return std::string(buf);
76
  }
77
 
78
  std::optional<std::string> normalize_path(const std::string &raw_path) {
 
9
  #include <ctime>
10
  #include <iomanip>
11
  #include <iostream>
12
+ #include <cerrno>
13
+ #include <fstream>
14
  #include <sstream>
15
  #include <stdexcept>
16
  #include <vector>
17
 
18
  #include <sys/socket.h>
19
  #include <sys/time.h>
20
+ #if defined(__linux__)
21
+ #include <sys/random.h>
22
+ #endif
23
 
24
  namespace asio = boost::asio;
25
  namespace beast = boost::beast;
 
59
  setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char *>(&snd), sizeof(snd));
60
  }
61
 
62
+ // Fills `out` from the operating system CSPRNG. Throws rather than falling
63
+ // back to anything predictable.
64
+ static void fill_os_random(unsigned char *out, size_t len) {
65
+ #if defined(__linux__)
66
+ size_t got = 0;
67
+ while (got < len) {
68
+ const ssize_t r = getrandom(out + got, len - got, 0);
69
+ if (r < 0) {
70
+ if (errno == EINTR) continue;
71
+ break; // fall through to /dev/urandom
72
+ }
73
+ got += static_cast<size_t>(r);
74
+ }
75
+ if (got == len) return;
76
+ #endif
77
+ std::ifstream urandom("/dev/urandom", std::ios::binary);
78
+ if (!urandom.read(reinterpret_cast<char *>(out), static_cast<std::streamsize>(len))) {
79
+ throw std::runtime_error("no OS random source available");
80
+ }
81
+ }
82
+
83
  std::string generate_request_id() {
84
+ unsigned char bytes[16];
85
+ fill_os_random(bytes, sizeof(bytes));
86
+ static const char *digits = "0123456789abcdef";
87
+ std::string out;
88
+ out.reserve(32);
89
+ for (unsigned char b : bytes) {
90
+ out.push_back(digits[b >> 4]);
91
+ out.push_back(digits[b & 0xF]);
 
 
 
92
  }
93
+ return out;
 
 
 
94
  }
95
 
96
  std::optional<std::string> normalize_path(const std::string &raw_path) {
cpp/llm_manager_types.h CHANGED
@@ -21,9 +21,14 @@ struct ServerConfig {
21
  // Hard cap on concurrently open client connections (each one pins a
22
  // thread). Excess connections get a fast 503.
23
  int max_sessions = 256;
24
- // Per-recv timeout for client sockets: bounds idle keep-alive connections
25
- // and stalled senders.
26
  int read_timeout_sec = 30;
 
 
 
 
 
 
27
  };
28
 
29
  struct WorkerConfig {
 
21
  // Hard cap on concurrently open client connections (each one pins a
22
  // thread). Excess connections get a fast 503.
23
  int max_sessions = 256;
24
+ // Idle timeout while waiting for the first byte of a request (keep-alive).
 
25
  int read_timeout_sec = 30;
26
+ // Absolute deadlines, measured from the first byte of a request: the full
27
+ // header must arrive within header_timeout_sec and the full body within
28
+ // body_timeout_sec, however slowly bytes trickle in. Exceeding them gets
29
+ // a 408 and the connection is closed.
30
+ int header_timeout_sec = 10;
31
+ int body_timeout_sec = 30;
32
  };
33
 
34
  struct WorkerConfig {
cpp/model_manager.cpp CHANGED
@@ -8,6 +8,7 @@
8
 
9
  #include <cerrno>
10
  #include <csignal>
 
11
  #include <thread>
12
  #include <unistd.h>
13
  #include <sys/wait.h>
@@ -225,16 +226,74 @@ pid_t ModelManager::spawn_worker(const std::string &model, int port) {
225
  for (auto &s : args) argv.push_back(const_cast<char *>(s.c_str()));
226
  argv.push_back(nullptr);
227
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
  pid_t pid = fork();
229
  if (pid < 0) return -1;
230
  if (pid == 0) {
231
  setsid();
232
- execvp(argv[0], argv.data());
233
  _exit(127);
234
  }
235
  return pid;
236
  }
237
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
238
  bool ModelManager::wait_until_ready(pid_t pid, int port, int timeout_sec) {
239
  const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec);
240
  // Exponential backoff: fast detection when the worker comes up (or dies)
 
8
 
9
  #include <cerrno>
10
  #include <csignal>
11
+ #include <cstdlib>
12
  #include <thread>
13
  #include <unistd.h>
14
  #include <sys/wait.h>
 
226
  for (auto &s : args) argv.push_back(const_cast<char *>(s.c_str()));
227
  argv.push_back(nullptr);
228
 
229
+ // The worker parses untrusted model files and serves user prompts; it must
230
+ // not inherit API_KEYS or any other manager secret. Hand it an explicit
231
+ // allowlisted environment instead of ours.
232
+ std::vector<std::string> env = worker_environment();
233
+ std::vector<char *> envp;
234
+ envp.reserve(env.size() + 1);
235
+ for (auto &s : env) envp.push_back(const_cast<char *>(s.c_str()));
236
+ envp.push_back(nullptr);
237
+
238
+ // execve does not search PATH; resolve a bare binary name up front.
239
+ const std::string resolved = resolve_executable(llama_server_bin_);
240
+ if (resolved.empty()) return -1;
241
+
242
  pid_t pid = fork();
243
  if (pid < 0) return -1;
244
  if (pid == 0) {
245
  setsid();
246
+ execve(resolved.c_str(), argv.data(), envp.data());
247
  _exit(127);
248
  }
249
  return pid;
250
  }
251
 
252
+ extern "C" char **environ;
253
+
254
+ // Variables the worker legitimately needs: locale, cache/home, the model
255
+ // download path (HF token for private repos, proxies, CA bundle) and
256
+ // llama.cpp/BLAS tuning knobs. Everything else, in particular API_KEYS and
257
+ // MANAGER_*, is dropped.
258
+ std::vector<std::string> ModelManager::worker_environment() {
259
+ static const char *kExact[] = {
260
+ "HOME", "PATH", "LANG", "TMPDIR", "TERM", "LD_LIBRARY_PATH", "LLAMA_CACHE",
261
+ "HF_TOKEN", "HF_HOME", "HF_HUB_CACHE", "HF_HUB_OFFLINE",
262
+ "CURL_CA_BUNDLE", "SSL_CERT_FILE", "SSL_CERT_DIR",
263
+ "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"};
264
+ static const char *kPrefixes[] = {"LC_", "LLAMA_", "GGML_", "OMP_", "OPENBLAS_", "HF_HUB_"};
265
+ std::vector<std::string> out;
266
+ for (char **e = environ; e && *e; ++e) {
267
+ const std::string entry(*e);
268
+ const auto eq = entry.find('=');
269
+ if (eq == std::string::npos) continue;
270
+ const std::string name = entry.substr(0, eq);
271
+ bool keep = false;
272
+ for (const char *n : kExact) if (name == n) { keep = true; break; }
273
+ for (const char *pfx : kPrefixes) if (!keep && name.rfind(pfx, 0) == 0) { keep = true; break; }
274
+ if (keep) out.push_back(entry);
275
+ }
276
+ return out;
277
+ }
278
+
279
+ std::string ModelManager::resolve_executable(const std::string &bin) {
280
+ if (bin.find('/') != std::string::npos) return bin;
281
+ const char *path = std::getenv("PATH");
282
+ std::string search = path ? path : "/usr/local/bin:/usr/bin:/bin";
283
+ size_t start = 0;
284
+ while (start <= search.size()) {
285
+ const auto end = search.find(':', start);
286
+ const std::string dir = search.substr(start, end == std::string::npos ? std::string::npos : end - start);
287
+ if (!dir.empty()) {
288
+ const std::string candidate = dir + "/" + bin;
289
+ if (access(candidate.c_str(), X_OK) == 0) return candidate;
290
+ }
291
+ if (end == std::string::npos) break;
292
+ start = end + 1;
293
+ }
294
+ return "";
295
+ }
296
+
297
  bool ModelManager::wait_until_ready(pid_t pid, int port, int timeout_sec) {
298
  const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec);
299
  // Exponential backoff: fast detection when the worker comes up (or dies)
cpp/model_manager.h CHANGED
@@ -4,6 +4,7 @@
4
  #include <optional>
5
  #include <string>
6
  #include <utility>
 
7
 
8
  #include "llm_manager_types.h"
9
 
@@ -40,6 +41,9 @@ private:
40
  int allocate_port();
41
  void finish_switch(bool ok);
42
  pid_t spawn_worker(const std::string &model, int port);
 
 
 
43
  bool wait_until_ready(pid_t pid, int port, int timeout_sec);
44
  std::pair<int, std::string> http_get(int port, const std::string &target);
45
  };
 
4
  #include <optional>
5
  #include <string>
6
  #include <utility>
7
+ #include <vector>
8
 
9
  #include "llm_manager_types.h"
10
 
 
41
  int allocate_port();
42
  void finish_switch(bool ok);
43
  pid_t spawn_worker(const std::string &model, int port);
44
+ // Allowlisted copy of our environment for the worker (no manager secrets).
45
+ static std::vector<std::string> worker_environment();
46
+ static std::string resolve_executable(const std::string &bin);
47
  bool wait_until_ready(pid_t pid, int port, int timeout_sec);
48
  std::pair<int, std::string> http_get(int port, const std::string &target);
49
  };
cpp/runtime_components.cpp CHANGED
@@ -19,8 +19,12 @@ RateLimitDecision RateLimiterStore::allow(const std::string &api_key_id, int est
19
  if (requests_per_minute_ <= 0 && estimated_tokens_per_minute_ <= 0) return {};
20
 
21
  std::lock_guard<std::mutex> lock(mu_);
22
- auto &bucket = buckets_[api_key_id];
23
  const auto now = std::chrono::steady_clock::now();
 
 
 
 
 
24
  refill(bucket.request_tokens, bucket.last_request_refill, requests_per_minute_, now);
25
  refill(bucket.estimated_tokens, bucket.last_estimated_refill, estimated_tokens_per_minute_, now);
26
 
@@ -47,6 +51,23 @@ RateLimitDecision RateLimiterStore::allow(const std::string &api_key_id, int est
47
  return {};
48
  }
49
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  void RateLimiterStore::refill(
51
  double &tokens,
52
  std::chrono::steady_clock::time_point &last_refill,
 
19
  if (requests_per_minute_ <= 0 && estimated_tokens_per_minute_ <= 0) return {};
20
 
21
  std::lock_guard<std::mutex> lock(mu_);
 
22
  const auto now = std::chrono::steady_clock::now();
23
+ if (buckets_.size() >= kMaxBuckets && buckets_.find(api_key_id) == buckets_.end()) {
24
+ prune_idle(now);
25
+ }
26
+ auto &bucket = buckets_[api_key_id];
27
+ bucket.last_seen = now;
28
  refill(bucket.request_tokens, bucket.last_request_refill, requests_per_minute_, now);
29
  refill(bucket.estimated_tokens, bucket.last_estimated_refill, estimated_tokens_per_minute_, now);
30
 
 
51
  return {};
52
  }
53
 
54
+ void RateLimiterStore::prune_idle(std::chrono::steady_clock::time_point now) {
55
+ for (auto it = buckets_.begin(); it != buckets_.end();) {
56
+ if (now - it->second.last_seen > kIdleEvict) it = buckets_.erase(it);
57
+ else ++it;
58
+ }
59
+ // Still full of active keys: drop the oldest ones so the map stays bounded
60
+ // even under a flood of fresh sources. Those sources merely get a fresh
61
+ // (full) bucket, which is no worse than before they were seen.
62
+ while (buckets_.size() >= kMaxBuckets) {
63
+ auto oldest = buckets_.begin();
64
+ for (auto it = buckets_.begin(); it != buckets_.end(); ++it) {
65
+ if (it->second.last_seen < oldest->second.last_seen) oldest = it;
66
+ }
67
+ buckets_.erase(oldest);
68
+ }
69
+ }
70
+
71
  void RateLimiterStore::refill(
72
  double &tokens,
73
  std::chrono::steady_clock::time_point &last_refill,
cpp/runtime_components.h CHANGED
@@ -25,13 +25,22 @@ private:
25
  double estimated_tokens = 0.0;
26
  std::chrono::steady_clock::time_point last_request_refill{};
27
  std::chrono::steady_clock::time_point last_estimated_refill{};
 
28
  };
29
 
 
 
 
 
 
 
30
  std::mutex mu_;
31
  std::unordered_map<std::string, Bucket> buckets_;
32
  int requests_per_minute_;
33
  int estimated_tokens_per_minute_;
34
 
 
 
35
  static void refill(
36
  double &tokens,
37
  std::chrono::steady_clock::time_point &last_refill,
 
25
  double estimated_tokens = 0.0;
26
  std::chrono::steady_clock::time_point last_request_refill{};
27
  std::chrono::steady_clock::time_point last_estimated_refill{};
28
+ std::chrono::steady_clock::time_point last_seen{};
29
  };
30
 
31
+ // Bound on distinct keys kept (per-IP limiting sees arbitrary many
32
+ // sources). Above this, buckets idle longer than kIdleEvict are dropped;
33
+ // an idle bucket is full anyway, so evicting it changes nothing.
34
+ static constexpr size_t kMaxBuckets = 10000;
35
+ static constexpr std::chrono::seconds kIdleEvict{120};
36
+
37
  std::mutex mu_;
38
  std::unordered_map<std::string, Bucket> buckets_;
39
  int requests_per_minute_;
40
  int estimated_tokens_per_minute_;
41
 
42
+ void prune_idle(std::chrono::steady_clock::time_point now);
43
+
44
  static void refill(
45
  double &tokens,
46
  std::chrono::steady_clock::time_point &last_refill,
cpp/server.cpp CHANGED
@@ -284,9 +284,10 @@ static bool is_allowed_model(const ModelsConfig &models, const std::string &mode
284
  return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
285
  }
286
 
287
- // Resolves the client address used for per-source rate limiting. The first
288
- // X-Forwarded-For hop is used only when the deployment says the proxy in
289
- // front of us is trusted (HF Spaces); otherwise the TCP peer.
 
290
  static std::string client_address(
291
  const http::request<http::string_body> &req,
292
  const std::string &peer_address,
@@ -295,10 +296,17 @@ static std::string client_address(
295
  const auto it = req.find("X-Forwarded-For");
296
  if (it != req.end()) {
297
  std::string value = it->value().to_string();
298
- const auto comma = value.find(',');
299
- if (comma != std::string::npos) value = value.substr(0, comma);
300
  value = trim_copy(value);
301
- if (!value.empty() && value.size() <= 64) return value;
 
 
 
 
 
 
 
302
  }
303
  }
304
  return peer_address;
@@ -432,14 +440,22 @@ http::response<http::string_body> handle_request(
432
  }
433
 
434
  if (path == "/health" && req.method() == http::verb::get) {
435
- return json_response(http::status::ok, manager.models_view());
 
 
 
 
 
 
436
  }
437
 
438
  if (path == "/models" && req.method() == http::verb::get) {
 
439
  return json_response(http::status::ok, manager.models_view());
440
  }
441
 
442
  if (path == "/queue/metrics" && req.method() == http::verb::get) {
 
443
  http::response<http::string_body> res{http::status::ok, req.version()};
444
  res.set(http::field::content_type, "text/plain; version=0.0.4; charset=utf-8");
445
  res.set(http::field::server, "llm-manager");
@@ -676,9 +692,12 @@ http::response<http::string_body> handle_request(
676
  " final_state=" + state_to_string(final_state) +
677
  " upstream_status=" + std::to_string(result.status) +
678
  " elapsed_ms=" + std::to_string(elapsed_ms));
 
 
679
  if (result.status >= 400) {
680
  log_line("request_id=" + request_id +
681
- " upstream_error_body=" + truncate_body(result.body));
 
682
  }
683
  return res;
684
  }
@@ -725,7 +744,8 @@ http::response<http::string_body> handle_request(
725
  " elapsed_ms=" + std::to_string(elapsed_ms));
726
  if (upstream.status >= 400) {
727
  log_line("request_id=" + request_id +
728
- " proxied_get_error_body=" + truncate_body(upstream.body));
 
729
  }
730
  return res;
731
  }
@@ -749,9 +769,7 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
749
  } session_guard{deps.active_sessions};
750
 
751
  const ManagerConfig &config = deps.config;
752
- // Receive timeout: each connection pins a detached thread, so idle
753
- // keep-alive connections and stalled senders must eventually release it.
754
- set_socket_timeouts(socket.native_handle(), config.server.read_timeout_sec, 60);
755
 
756
  std::string peer_address;
757
  {
@@ -760,6 +778,17 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
760
  if (!ec) peer_address = endpoint.address().to_string();
761
  }
762
 
 
 
 
 
 
 
 
 
 
 
 
763
  try {
764
  beast::flat_buffer buffer;
765
  for (;;) {
@@ -768,17 +797,58 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
768
  http::request_parser<http::string_body> parser;
769
  parser.header_limit(16 * 1024);
770
  parser.body_limit(static_cast<uint64_t>(config.limits.max_request_body_bytes));
 
 
 
 
 
 
 
771
  beast::error_code read_ec;
772
- http::read(socket, buffer, parser, read_ec);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
773
  if (read_ec == http::error::body_limit || read_ec == http::error::header_limit) {
774
- http::response<http::string_body> res{http::status::payload_too_large, 11};
775
- res.set(http::field::content_type, "application/json");
776
- res.set(http::field::server, "llm-manager");
777
- res.keep_alive(false);
778
- res.body() = R"({"error":"Request too large"})";
779
- res.prepare_payload();
780
- beast::error_code write_ec;
781
- http::write(socket, res, write_ec);
782
  break;
783
  }
784
  if (read_ec) throw beast::system_error{read_ec};
@@ -797,6 +867,7 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
797
  // would_block/try_again family, which is the SO_RCVTIMEO receive timeout
798
  // firing on an idle keep-alive connection. Neither is a real error.
799
  if (e.code() != http::error::end_of_stream &&
 
800
  e.code() != boost::asio::error::would_block &&
801
  e.code() != boost::asio::error::try_again &&
802
  e.code() != boost::asio::error::timed_out)
 
284
  return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
285
  }
286
 
287
+ // Resolves the client address used for per-source rate limiting. Behind a
288
+ // trusted reverse proxy (HF Spaces) the proxy appends the real client to
289
+ // X-Forwarded-For, so the LAST hop is the one it vouches for; anything
290
+ // before it was supplied by the client and must not be trusted.
291
  static std::string client_address(
292
  const http::request<http::string_body> &req,
293
  const std::string &peer_address,
 
296
  const auto it = req.find("X-Forwarded-For");
297
  if (it != req.end()) {
298
  std::string value = it->value().to_string();
299
+ const auto comma = value.rfind(',');
300
+ if (comma != std::string::npos) value = value.substr(comma + 1);
301
  value = trim_copy(value);
302
+ // Only accept something that looks like an address: digits, hex,
303
+ // dots and colons. Anything else falls back to the TCP peer so a
304
+ // client cannot pick an arbitrary bucket key.
305
+ const bool address_like = !value.empty() && value.size() <= 45 &&
306
+ std::all_of(value.begin(), value.end(), [](unsigned char c) {
307
+ return std::isxdigit(c) || c == '.' || c == ':';
308
+ });
309
+ if (address_like) return value;
310
  }
311
  }
312
  return peer_address;
 
440
  }
441
 
442
  if (path == "/health" && req.method() == http::verb::get) {
443
+ // Public liveness only: which model is loaded, counters and
444
+ // internals are behind a key (/models, /queue/metrics).
445
+ const auto view = manager.models_view();
446
+ return json_response(http::status::ok, {
447
+ {"status", view.value("status", "unknown")},
448
+ {"switch_in_progress", view.value("switch_in_progress", false)}
449
+ });
450
  }
451
 
452
  if (path == "/models" && req.method() == http::verb::get) {
453
+ if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
454
  return json_response(http::status::ok, manager.models_view());
455
  }
456
 
457
  if (path == "/queue/metrics" && req.method() == http::verb::get) {
458
+ if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
459
  http::response<http::string_body> res{http::status::ok, req.version()};
460
  res.set(http::field::content_type, "text/plain; version=0.0.4; charset=utf-8");
461
  res.set(http::field::server, "llm-manager");
 
692
  " final_state=" + state_to_string(final_state) +
693
  " upstream_status=" + std::to_string(result.status) +
694
  " elapsed_ms=" + std::to_string(elapsed_ms));
695
+ // Upstream error bodies can echo user content; status and size
696
+ // are enough unless body logging is explicitly on.
697
  if (result.status >= 400) {
698
  log_line("request_id=" + request_id +
699
+ " upstream_error_bytes=" + std::to_string(result.body.size()) +
700
+ (config.logging.log_request_bodies ? " upstream_error_body=" + truncate_body(result.body) : ""));
701
  }
702
  return res;
703
  }
 
744
  " elapsed_ms=" + std::to_string(elapsed_ms));
745
  if (upstream.status >= 400) {
746
  log_line("request_id=" + request_id +
747
+ " proxied_get_error_bytes=" + std::to_string(upstream.body.size()) +
748
+ (config.logging.log_request_bodies ? " proxied_get_error_body=" + truncate_body(upstream.body) : ""));
749
  }
750
  return res;
751
  }
 
769
  } session_guard{deps.active_sessions};
770
 
771
  const ManagerConfig &config = deps.config;
772
+ using clock = std::chrono::steady_clock;
 
 
773
 
774
  std::string peer_address;
775
  {
 
778
  if (!ec) peer_address = endpoint.address().to_string();
779
  }
780
 
781
+ auto write_and_close = [&](http::status status, const char *body) {
782
+ http::response<http::string_body> res{status, 11};
783
+ res.set(http::field::content_type, "application/json");
784
+ res.set(http::field::server, "llm-manager");
785
+ res.keep_alive(false);
786
+ res.body() = body;
787
+ res.prepare_payload();
788
+ beast::error_code write_ec;
789
+ http::write(socket, res, write_ec);
790
+ };
791
+
792
  try {
793
  beast::flat_buffer buffer;
794
  for (;;) {
 
797
  http::request_parser<http::string_body> parser;
798
  parser.header_limit(16 * 1024);
799
  parser.body_limit(static_cast<uint64_t>(config.limits.max_request_body_bytes));
800
+
801
+ // Read in pieces so an absolute deadline can be enforced. Before
802
+ // the first byte only the idle timeout applies (keep-alive); from
803
+ // the first byte on, the header and then the whole message must
804
+ // complete within fixed budgets regardless of how slowly bytes
805
+ // arrive. SO_RCVTIMEO is re-armed each iteration to the smaller of
806
+ // the idle timeout and the time left on the deadline.
807
  beast::error_code read_ec;
808
+ clock::time_point first_byte{};
809
+ bool deadline_exceeded = false;
810
+ bool idle_closed = false;
811
+ while (!parser.is_done()) {
812
+ const auto now = clock::now();
813
+ long wait_sec = config.server.read_timeout_sec;
814
+ if (first_byte == clock::time_point{} && (parser.got_some() || buffer.size() > 0)) {
815
+ first_byte = now;
816
+ }
817
+ if (first_byte != clock::time_point{}) {
818
+ const int budget = parser.is_header_done() ? config.server.body_timeout_sec
819
+ : config.server.header_timeout_sec;
820
+ const auto limit = first_byte + std::chrono::seconds(budget);
821
+ if (now >= limit) {
822
+ deadline_exceeded = true;
823
+ break;
824
+ }
825
+ const long remaining = std::chrono::duration_cast<std::chrono::seconds>(limit - now).count() + 1;
826
+ wait_sec = std::min(wait_sec, remaining);
827
+ }
828
+ set_socket_timeouts(socket.native_handle(), static_cast<int>(std::max(1L, wait_sec)), 60);
829
+
830
+ read_ec = {};
831
+ http::read_some(socket, buffer, parser, read_ec);
832
+ if (read_ec == boost::asio::error::would_block ||
833
+ read_ec == boost::asio::error::try_again ||
834
+ read_ec == boost::asio::error::timed_out) {
835
+ if (first_byte == clock::time_point{}) {
836
+ idle_closed = true; // nothing started; plain keep-alive expiry
837
+ break;
838
+ }
839
+ continue; // deadline is re-checked at the top of the loop
840
+ }
841
+ if (read_ec) break;
842
+ }
843
+
844
+ if (idle_closed) break;
845
+ if (deadline_exceeded) {
846
+ log_line("session: request read deadline exceeded peer=" + peer_address);
847
+ write_and_close(http::status::request_timeout, R"({"error":"Request read timed out"})");
848
+ break;
849
+ }
850
  if (read_ec == http::error::body_limit || read_ec == http::error::header_limit) {
851
+ write_and_close(http::status::payload_too_large, R"({"error":"Request too large"})");
 
 
 
 
 
 
 
852
  break;
853
  }
854
  if (read_ec) throw beast::system_error{read_ec};
 
867
  // would_block/try_again family, which is the SO_RCVTIMEO receive timeout
868
  // firing on an idle keep-alive connection. Neither is a real error.
869
  if (e.code() != http::error::end_of_stream &&
870
+ e.code() != http::error::partial_message &&
871
  e.code() != boost::asio::error::would_block &&
872
  e.code() != boost::asio::error::try_again &&
873
  e.code() != boost::asio::error::timed_out)
python/app.py CHANGED
@@ -22,6 +22,16 @@ from logger import get_logger
22
 
23
  logger = get_logger(__name__)
24
 
 
 
 
 
 
 
 
 
 
 
25
  app = FastAPI(
26
  title="AGI Multi-Model API",
27
  description="""
@@ -110,7 +120,7 @@ BASE_PORT = int(os.getenv("BASE_PORT", "8080"))
110
  PRELOAD_MODELS = os.getenv("PRELOAD_MODELS", "").split(",") if os.getenv("PRELOAD_MODELS") else []
111
  WEB_SEARCH_CACHE_TTL = int(os.getenv("WEB_SEARCH_CACHE_TTL", "3600")) # 1 hour
112
  REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "300")) # 5 minutes
113
- LOG_REQUEST_BODY = os.getenv("LOG_REQUEST_BODY", "1") == "1"
114
  LOG_REQUEST_BODY_MAX_CHARS = int(os.getenv("LOG_REQUEST_BODY_MAX_CHARS", "2000"))
115
  CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "2048"))
116
 
 
22
 
23
  logger = get_logger(__name__)
24
 
25
+ # This legacy app has NO authentication: /switch-model, /cache/clear and the
26
+ # chat routes are public, and web-chat forwards queries to an external search
27
+ # provider. It is kept for reference only and is not part of the Docker image.
28
+ # Refuse to start unless the operator explicitly acknowledges that.
29
+ if os.getenv("LEGACY_UNAUTHENTICATED_APP_ACK") != "1":
30
+ raise RuntimeError(
31
+ "python/app.py is the unauthenticated legacy service. Use the C++ llm-manager, "
32
+ "or set LEGACY_UNAUTHENTICATED_APP_ACK=1 to run this on a trusted network only."
33
+ )
34
+
35
  app = FastAPI(
36
  title="AGI Multi-Model API",
37
  description="""
 
120
  PRELOAD_MODELS = os.getenv("PRELOAD_MODELS", "").split(",") if os.getenv("PRELOAD_MODELS") else []
121
  WEB_SEARCH_CACHE_TTL = int(os.getenv("WEB_SEARCH_CACHE_TTL", "3600")) # 1 hour
122
  REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "300")) # 5 minutes
123
+ LOG_REQUEST_BODY = os.getenv("LOG_REQUEST_BODY", "0") == "1" # prompts are user data; opt-in only
124
  LOG_REQUEST_BODY_MAX_CHARS = int(os.getenv("LOG_REQUEST_BODY_MAX_CHARS", "2000"))
125
  CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "2048"))
126