Spaces:
Sleeping
Sleeping
Dmitry Beresnev commited on
Commit ·
70dee29
1
Parent(s): 6dc320d
fix security volnurabilities
Browse files- Dockerfile +21 -7
- README.md +3 -2
- cpp/config.cpp +9 -0
- cpp/http_helpers.cpp +35 -17
- cpp/llm_manager_types.h +7 -2
- cpp/model_manager.cpp +60 -1
- cpp/model_manager.h +4 -0
- cpp/runtime_components.cpp +22 -1
- cpp/runtime_components.h +9 -0
- cpp/server.cpp +92 -21
- python/app.py +11 -1
Dockerfile
CHANGED
|
@@ -19,14 +19,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
| 19 |
# Clone and build llama.cpp with SSL support for HuggingFace Hub
|
| 20 |
WORKDIR /build
|
| 21 |
ARG CACHEBUST=6
|
| 22 |
-
# Pinned
|
| 23 |
-
# deliberately and re-run the endpoint tests.
|
| 24 |
-
ARG LLAMA_CPP_REF=
|
| 25 |
#ARG BUILD_PROFILE=fast_build
|
| 26 |
ARG BUILD_PROFILE=fast_runtime
|
| 27 |
ARG BUILD_JOBS=1
|
| 28 |
-
RUN
|
| 29 |
-
|
|
|
|
|
|
|
|
|
|
| 30 |
if [ "${BUILD_PROFILE}" = "fast_runtime" ]; then \
|
| 31 |
C_FLAGS="-O3 -DNDEBUG"; \
|
| 32 |
CXX_FLAGS="-O3 -DNDEBUG"; \
|
|
@@ -113,6 +116,12 @@ ENV HOME=/home/user \
|
|
| 113 |
|
| 114 |
EXPOSE 7860
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
# Start FastAPI app (which manages llama-server internally)
|
| 117 |
#CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
| 118 |
|
|
@@ -120,6 +129,7 @@ EXPOSE 7860
|
|
| 120 |
# Keep the FastAPI CMD above as the default. Uncomment ONE of the following
|
| 121 |
# to run the C++ server directly instead of the Python app.
|
| 122 |
#
|
|
|
|
| 123 |
# Example DeepSeek (4k context):
|
| 124 |
# CMD ["llama-server", "-hf", "TheBloke/deepseek-llm-7B-chat-GGUF:deepseek-llm-7b-chat.Q4_K_M.gguf",
|
| 125 |
# "--host", "0.0.0.0", "--port", "7860", "-c", "4096", "-t", "4", "-ngl", "0", "--cont-batching", "-b", "512"]
|
|
@@ -150,7 +160,11 @@ EXPOSE 7860
|
|
| 150 |
# - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
|
| 151 |
# (X-Forwarded-For from the HF proxy).
|
| 152 |
# - MIN_FREE_DISK_MB refuses a model switch when the cache volume is nearly full.
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
PER_IP_REQUESTS_PER_MINUTE=120 \
|
| 155 |
MANAGER_MAX_SESSIONS=256 \
|
| 156 |
MAX_REQUEST_BODY_BYTES=1048576 \
|
|
@@ -159,7 +173,7 @@ ENV DEFAULT_MODEL=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
|
|
| 159 |
MANAGER_HOST=0.0.0.0 \
|
| 160 |
MANAGER_PORT=7860 \
|
| 161 |
WORKER_BASE_PORT=8080 \
|
| 162 |
-
SWITCH_TIMEOUT_SEC=
|
| 163 |
REQUEST_TIMEOUT_SEC=300 \
|
| 164 |
DEFAULT_MAX_TOKENS=2048 \
|
| 165 |
MAX_TOKENS_PER_REQUEST=4096 \
|
|
|
|
| 19 |
# Clone and build llama.cpp with SSL support for HuggingFace Hub
|
| 20 |
WORKDIR /build
|
| 21 |
ARG CACHEBUST=6
|
| 22 |
+
# Pinned to an immutable commit (release tag b12367). Tags can be moved;
|
| 23 |
+
# commits cannot. Bump deliberately and re-run the endpoint tests.
|
| 24 |
+
ARG LLAMA_CPP_REF=fe14cad51bbfb3f1cc8d9cfd9d70e98a123743d3
|
| 25 |
#ARG BUILD_PROFILE=fast_build
|
| 26 |
ARG BUILD_PROFILE=fast_runtime
|
| 27 |
ARG BUILD_JOBS=1
|
| 28 |
+
RUN mkdir llama.cpp && cd llama.cpp && git init -q && \
|
| 29 |
+
git remote add origin https://github.com/ggerganov/llama.cpp.git && \
|
| 30 |
+
git fetch --depth 1 origin ${LLAMA_CPP_REF} && \
|
| 31 |
+
git checkout -q FETCH_HEAD && \
|
| 32 |
+
echo "llama.cpp commit: $(git rev-parse HEAD)" && \
|
| 33 |
if [ "${BUILD_PROFILE}" = "fast_runtime" ]; then \
|
| 34 |
C_FLAGS="-O3 -DNDEBUG"; \
|
| 35 |
CXX_FLAGS="-O3 -DNDEBUG"; \
|
|
|
|
| 116 |
|
| 117 |
EXPOSE 7860
|
| 118 |
|
| 119 |
+
# The only supported entrypoint is the C++ manager (CMD at the bottom). The
|
| 120 |
+
# legacy FastAPI app is unauthenticated and is not in this image.
|
| 121 |
+
#
|
| 122 |
+
# Historical examples of running llama-server directly (unauthenticated —
|
| 123 |
+
# do not use in production):
|
| 124 |
+
|
| 125 |
# Start FastAPI app (which manages llama-server internally)
|
| 126 |
#CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
| 127 |
|
|
|
|
| 129 |
# Keep the FastAPI CMD above as the default. Uncomment ONE of the following
|
| 130 |
# to run the C++ server directly instead of the Python app.
|
| 131 |
#
|
| 132 |
+
|
| 133 |
# Example DeepSeek (4k context):
|
| 134 |
# CMD ["llama-server", "-hf", "TheBloke/deepseek-llm-7B-chat-GGUF:deepseek-llm-7b-chat.Q4_K_M.gguf",
|
| 135 |
# "--host", "0.0.0.0", "--port", "7860", "-c", "4096", "-t", "4", "-ngl", "0", "--cont-batching", "-b", "512"]
|
|
|
|
| 160 |
# - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
|
| 161 |
# (X-Forwarded-For from the HF proxy).
|
| 162 |
# - MIN_FREE_DISK_MB refuses a model switch when the cache volume is nearly full.
|
| 163 |
+
# - The allowlist below is the curated set from docs/MODEL_CANDIDATES_FREE_SPACE.md:
|
| 164 |
+
# dense upgrades (Qwen3.5), MoE fast lanes (LFM2.5-8B-A1B, Granite 4.0 H-Tiny,
|
| 165 |
+
# 1-1.5B active), Gemma 4, and one large MoE (Qwen3.6-35B-A3B at IQ3, 12.3 GiB)
|
| 166 |
+
# that needs SWITCH_TIMEOUT_SEC >= 600 on a cold start.
|
| 167 |
+
ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m,unsloth/Qwen3.5-9B-GGUF:Q4_K_M,unsloth/Qwen3.5-4B-GGUF:Q4_K_M,LiquidAI/LFM2.5-8B-A1B-GGUF:Q4_K_M,unsloth/granite-4.0-h-tiny-GGUF:Q4_K_M,unsloth/gemma-4-E4B-it-GGUF:Q4_K_M,unsloth/gemma-4-E2B-it-GGUF:Q4_K_M,unsloth/Qwen3.6-35B-A3B-GGUF:UD-IQ3_XXS \
|
| 168 |
PER_IP_REQUESTS_PER_MINUTE=120 \
|
| 169 |
MANAGER_MAX_SESSIONS=256 \
|
| 170 |
MAX_REQUEST_BODY_BYTES=1048576 \
|
|
|
|
| 173 |
MANAGER_HOST=0.0.0.0 \
|
| 174 |
MANAGER_PORT=7860 \
|
| 175 |
WORKER_BASE_PORT=8080 \
|
| 176 |
+
SWITCH_TIMEOUT_SEC=600 \
|
| 177 |
REQUEST_TIMEOUT_SEC=300 \
|
| 178 |
DEFAULT_MAX_TOKENS=2048 \
|
| 179 |
MAX_TOKENS_PER_REQUEST=4096 \
|
README.md
CHANGED
|
@@ -28,7 +28,8 @@ The legacy Python application is still present under [`python/`](/Users/dmitrybe
|
|
| 28 |
## Security Setup (read before deploying)
|
| 29 |
|
| 30 |
The manager **fails closed**: with no usable API keys configured every authenticated endpoint
|
| 31 |
-
(`/v1/chat/completions`, `/switch-model`, `/stop`, cancel, `/
|
|
|
|
| 32 |
Keys are never baked into the image. Configure them as an environment variable / HF Space secret:
|
| 33 |
|
| 34 |
```bash
|
|
@@ -212,7 +213,7 @@ Roles:
|
|
| 212 |
- `admin`: can call privileged control endpoints
|
| 213 |
- `user`: can submit chat requests and cancel owned requests
|
| 214 |
|
| 215 |
-
If no API keys are configured
|
| 216 |
|
| 217 |
## Configuration
|
| 218 |
|
|
|
|
| 28 |
## Security Setup (read before deploying)
|
| 29 |
|
| 30 |
The manager **fails closed**: with no usable API keys configured every authenticated endpoint
|
| 31 |
+
(`/v1/chat/completions`, `/switch-model`, `/stop`, cancel, `/models`, `/queue/metrics`, `/props`, `/slots`,
|
| 32 |
+
`/metrics`) returns 401. Only `/health` (liveness: status and switch flag) and the static web UI are public.
|
| 33 |
Keys are never baked into the image. Configure them as an environment variable / HF Space secret:
|
| 34 |
|
| 35 |
```bash
|
|
|
|
| 213 |
- `admin`: can call privileged control endpoints
|
| 214 |
- `user`: can submit chat requests and cancel owned requests
|
| 215 |
|
| 216 |
+
If no usable API keys are configured the manager **fails closed**: every protected endpoint returns 401 (only `/health` and the static web UI stay public). Set `AUTH_ALLOW_ANONYMOUS=true` explicitly to allow unauthenticated callers with the user role.
|
| 217 |
|
| 218 |
## Configuration
|
| 219 |
|
cpp/config.cpp
CHANGED
|
@@ -255,6 +255,7 @@ static bool is_hex_digest(const std::string &value) {
|
|
| 255 |
static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig &auth) {
|
| 256 |
static const char *kPlaceholders[] = {"change-me", "changeme", "replace", "secret", "password", "test", "example"};
|
| 257 |
std::unordered_set<std::string> seen_ids;
|
|
|
|
| 258 |
for (auto &record : keys) {
|
| 259 |
if (!seen_ids.insert(record.key_id).second) {
|
| 260 |
log_line("config: duplicate api key id '" + record.key_id + "' — disabling later entry");
|
|
@@ -289,6 +290,12 @@ static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig
|
|
| 289 |
record.enabled = false;
|
| 290 |
}
|
| 291 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
}
|
| 293 |
}
|
| 294 |
|
|
@@ -308,6 +315,8 @@ ManagerConfig load_manager_config() {
|
|
| 308 |
cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
|
| 309 |
cfg.server.max_sessions = std::max(1, get_env_int_or("MANAGER_MAX_SESSIONS", get_toml_int_or(toml, "server", "max_sessions", cfg.server.max_sessions)));
|
| 310 |
cfg.server.read_timeout_sec = std::max(1, get_env_int_or("MANAGER_READ_TIMEOUT_SEC", get_toml_int_or(toml, "server", "read_timeout_sec", cfg.server.read_timeout_sec)));
|
|
|
|
|
|
|
| 311 |
|
| 312 |
cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
|
| 313 |
cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
|
|
|
|
| 255 |
static void finalize_api_keys(std::vector<ApiKeyRecord> &keys, const AuthConfig &auth) {
|
| 256 |
static const char *kPlaceholders[] = {"change-me", "changeme", "replace", "secret", "password", "test", "example"};
|
| 257 |
std::unordered_set<std::string> seen_ids;
|
| 258 |
+
std::unordered_set<std::string> seen_digests;
|
| 259 |
for (auto &record : keys) {
|
| 260 |
if (!seen_ids.insert(record.key_id).second) {
|
| 261 |
log_line("config: duplicate api key id '" + record.key_id + "' — disabling later entry");
|
|
|
|
| 290 |
record.enabled = false;
|
| 291 |
}
|
| 292 |
}
|
| 293 |
+
// The same secret under two ids (possibly two roles) is ambiguous:
|
| 294 |
+
// whichever loaded first would silently win. Fail closed on the rest.
|
| 295 |
+
if (record.enabled && !record.secret_sha256.empty() && !seen_digests.insert(record.secret_sha256).second) {
|
| 296 |
+
log_line("config: api key '" + record.key_id + "' reuses another key's secret — disabled");
|
| 297 |
+
record.enabled = false;
|
| 298 |
+
}
|
| 299 |
}
|
| 300 |
}
|
| 301 |
|
|
|
|
| 315 |
cfg.server.port = get_env_int_or("MANAGER_PORT", get_toml_int_or(toml, "server", "port", cfg.server.port));
|
| 316 |
cfg.server.max_sessions = std::max(1, get_env_int_or("MANAGER_MAX_SESSIONS", get_toml_int_or(toml, "server", "max_sessions", cfg.server.max_sessions)));
|
| 317 |
cfg.server.read_timeout_sec = std::max(1, get_env_int_or("MANAGER_READ_TIMEOUT_SEC", get_toml_int_or(toml, "server", "read_timeout_sec", cfg.server.read_timeout_sec)));
|
| 318 |
+
cfg.server.header_timeout_sec = std::max(1, get_env_int_or("MANAGER_HEADER_TIMEOUT_SEC", get_toml_int_or(toml, "server", "header_timeout_sec", cfg.server.header_timeout_sec)));
|
| 319 |
+
cfg.server.body_timeout_sec = std::max(cfg.server.header_timeout_sec, get_env_int_or("MANAGER_BODY_TIMEOUT_SEC", get_toml_int_or(toml, "server", "body_timeout_sec", cfg.server.body_timeout_sec)));
|
| 320 |
|
| 321 |
cfg.worker.default_model = get_env_or("DEFAULT_MODEL", get_toml_string_or(toml, "worker", "default_model", cfg.worker.default_model));
|
| 322 |
cfg.worker.llama_server_bin = get_env_or("LLAMA_SERVER_BIN", get_toml_string_or(toml, "worker", "llama_server_bin", cfg.worker.llama_server_bin));
|
cpp/http_helpers.cpp
CHANGED
|
@@ -9,14 +9,17 @@
|
|
| 9 |
#include <ctime>
|
| 10 |
#include <iomanip>
|
| 11 |
#include <iostream>
|
| 12 |
-
#include <
|
| 13 |
-
#include <
|
| 14 |
#include <sstream>
|
| 15 |
#include <stdexcept>
|
| 16 |
#include <vector>
|
| 17 |
|
| 18 |
#include <sys/socket.h>
|
| 19 |
#include <sys/time.h>
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
namespace asio = boost::asio;
|
| 22 |
namespace beast = boost::beast;
|
|
@@ -56,23 +59,38 @@ void set_socket_timeouts(int fd, int recv_sec, int send_sec) {
|
|
| 56 |
setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char *>(&snd), sizeof(snd));
|
| 57 |
}
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
std::string generate_request_id() {
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
std::lock_guard<std::mutex> lock(mu);
|
| 69 |
-
hi = rng();
|
| 70 |
-
lo = rng();
|
| 71 |
}
|
| 72 |
-
|
| 73 |
-
std::snprintf(buf, sizeof(buf), "%016llx%016llx",
|
| 74 |
-
static_cast<unsigned long long>(hi), static_cast<unsigned long long>(lo));
|
| 75 |
-
return std::string(buf);
|
| 76 |
}
|
| 77 |
|
| 78 |
std::optional<std::string> normalize_path(const std::string &raw_path) {
|
|
|
|
| 9 |
#include <ctime>
|
| 10 |
#include <iomanip>
|
| 11 |
#include <iostream>
|
| 12 |
+
#include <cerrno>
|
| 13 |
+
#include <fstream>
|
| 14 |
#include <sstream>
|
| 15 |
#include <stdexcept>
|
| 16 |
#include <vector>
|
| 17 |
|
| 18 |
#include <sys/socket.h>
|
| 19 |
#include <sys/time.h>
|
| 20 |
+
#if defined(__linux__)
|
| 21 |
+
#include <sys/random.h>
|
| 22 |
+
#endif
|
| 23 |
|
| 24 |
namespace asio = boost::asio;
|
| 25 |
namespace beast = boost::beast;
|
|
|
|
| 59 |
setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char *>(&snd), sizeof(snd));
|
| 60 |
}
|
| 61 |
|
| 62 |
+
// Fills `out` from the operating system CSPRNG. Throws rather than falling
|
| 63 |
+
// back to anything predictable.
|
| 64 |
+
static void fill_os_random(unsigned char *out, size_t len) {
|
| 65 |
+
#if defined(__linux__)
|
| 66 |
+
size_t got = 0;
|
| 67 |
+
while (got < len) {
|
| 68 |
+
const ssize_t r = getrandom(out + got, len - got, 0);
|
| 69 |
+
if (r < 0) {
|
| 70 |
+
if (errno == EINTR) continue;
|
| 71 |
+
break; // fall through to /dev/urandom
|
| 72 |
+
}
|
| 73 |
+
got += static_cast<size_t>(r);
|
| 74 |
+
}
|
| 75 |
+
if (got == len) return;
|
| 76 |
+
#endif
|
| 77 |
+
std::ifstream urandom("/dev/urandom", std::ios::binary);
|
| 78 |
+
if (!urandom.read(reinterpret_cast<char *>(out), static_cast<std::streamsize>(len))) {
|
| 79 |
+
throw std::runtime_error("no OS random source available");
|
| 80 |
+
}
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
std::string generate_request_id() {
|
| 84 |
+
unsigned char bytes[16];
|
| 85 |
+
fill_os_random(bytes, sizeof(bytes));
|
| 86 |
+
static const char *digits = "0123456789abcdef";
|
| 87 |
+
std::string out;
|
| 88 |
+
out.reserve(32);
|
| 89 |
+
for (unsigned char b : bytes) {
|
| 90 |
+
out.push_back(digits[b >> 4]);
|
| 91 |
+
out.push_back(digits[b & 0xF]);
|
|
|
|
|
|
|
|
|
|
| 92 |
}
|
| 93 |
+
return out;
|
|
|
|
|
|
|
|
|
|
| 94 |
}
|
| 95 |
|
| 96 |
std::optional<std::string> normalize_path(const std::string &raw_path) {
|
cpp/llm_manager_types.h
CHANGED
|
@@ -21,9 +21,14 @@ struct ServerConfig {
|
|
| 21 |
// Hard cap on concurrently open client connections (each one pins a
|
| 22 |
// thread). Excess connections get a fast 503.
|
| 23 |
int max_sessions = 256;
|
| 24 |
-
//
|
| 25 |
-
// and stalled senders.
|
| 26 |
int read_timeout_sec = 30;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
};
|
| 28 |
|
| 29 |
struct WorkerConfig {
|
|
|
|
| 21 |
// Hard cap on concurrently open client connections (each one pins a
|
| 22 |
// thread). Excess connections get a fast 503.
|
| 23 |
int max_sessions = 256;
|
| 24 |
+
// Idle timeout while waiting for the first byte of a request (keep-alive).
|
|
|
|
| 25 |
int read_timeout_sec = 30;
|
| 26 |
+
// Absolute deadlines, measured from the first byte of a request: the full
|
| 27 |
+
// header must arrive within header_timeout_sec and the full body within
|
| 28 |
+
// body_timeout_sec, however slowly bytes trickle in. Exceeding them gets
|
| 29 |
+
// a 408 and the connection is closed.
|
| 30 |
+
int header_timeout_sec = 10;
|
| 31 |
+
int body_timeout_sec = 30;
|
| 32 |
};
|
| 33 |
|
| 34 |
struct WorkerConfig {
|
cpp/model_manager.cpp
CHANGED
|
@@ -8,6 +8,7 @@
|
|
| 8 |
|
| 9 |
#include <cerrno>
|
| 10 |
#include <csignal>
|
|
|
|
| 11 |
#include <thread>
|
| 12 |
#include <unistd.h>
|
| 13 |
#include <sys/wait.h>
|
|
@@ -225,16 +226,74 @@ pid_t ModelManager::spawn_worker(const std::string &model, int port) {
|
|
| 225 |
for (auto &s : args) argv.push_back(const_cast<char *>(s.c_str()));
|
| 226 |
argv.push_back(nullptr);
|
| 227 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
pid_t pid = fork();
|
| 229 |
if (pid < 0) return -1;
|
| 230 |
if (pid == 0) {
|
| 231 |
setsid();
|
| 232 |
-
|
| 233 |
_exit(127);
|
| 234 |
}
|
| 235 |
return pid;
|
| 236 |
}
|
| 237 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
bool ModelManager::wait_until_ready(pid_t pid, int port, int timeout_sec) {
|
| 239 |
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec);
|
| 240 |
// Exponential backoff: fast detection when the worker comes up (or dies)
|
|
|
|
| 8 |
|
| 9 |
#include <cerrno>
|
| 10 |
#include <csignal>
|
| 11 |
+
#include <cstdlib>
|
| 12 |
#include <thread>
|
| 13 |
#include <unistd.h>
|
| 14 |
#include <sys/wait.h>
|
|
|
|
| 226 |
for (auto &s : args) argv.push_back(const_cast<char *>(s.c_str()));
|
| 227 |
argv.push_back(nullptr);
|
| 228 |
|
| 229 |
+
// The worker parses untrusted model files and serves user prompts; it must
|
| 230 |
+
// not inherit API_KEYS or any other manager secret. Hand it an explicit
|
| 231 |
+
// allowlisted environment instead of ours.
|
| 232 |
+
std::vector<std::string> env = worker_environment();
|
| 233 |
+
std::vector<char *> envp;
|
| 234 |
+
envp.reserve(env.size() + 1);
|
| 235 |
+
for (auto &s : env) envp.push_back(const_cast<char *>(s.c_str()));
|
| 236 |
+
envp.push_back(nullptr);
|
| 237 |
+
|
| 238 |
+
// execve does not search PATH; resolve a bare binary name up front.
|
| 239 |
+
const std::string resolved = resolve_executable(llama_server_bin_);
|
| 240 |
+
if (resolved.empty()) return -1;
|
| 241 |
+
|
| 242 |
pid_t pid = fork();
|
| 243 |
if (pid < 0) return -1;
|
| 244 |
if (pid == 0) {
|
| 245 |
setsid();
|
| 246 |
+
execve(resolved.c_str(), argv.data(), envp.data());
|
| 247 |
_exit(127);
|
| 248 |
}
|
| 249 |
return pid;
|
| 250 |
}
|
| 251 |
|
| 252 |
+
extern "C" char **environ;
|
| 253 |
+
|
| 254 |
+
// Variables the worker legitimately needs: locale, cache/home, the model
|
| 255 |
+
// download path (HF token for private repos, proxies, CA bundle) and
|
| 256 |
+
// llama.cpp/BLAS tuning knobs. Everything else, in particular API_KEYS and
|
| 257 |
+
// MANAGER_*, is dropped.
|
| 258 |
+
std::vector<std::string> ModelManager::worker_environment() {
|
| 259 |
+
static const char *kExact[] = {
|
| 260 |
+
"HOME", "PATH", "LANG", "TMPDIR", "TERM", "LD_LIBRARY_PATH", "LLAMA_CACHE",
|
| 261 |
+
"HF_TOKEN", "HF_HOME", "HF_HUB_CACHE", "HF_HUB_OFFLINE",
|
| 262 |
+
"CURL_CA_BUNDLE", "SSL_CERT_FILE", "SSL_CERT_DIR",
|
| 263 |
+
"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"};
|
| 264 |
+
static const char *kPrefixes[] = {"LC_", "LLAMA_", "GGML_", "OMP_", "OPENBLAS_", "HF_HUB_"};
|
| 265 |
+
std::vector<std::string> out;
|
| 266 |
+
for (char **e = environ; e && *e; ++e) {
|
| 267 |
+
const std::string entry(*e);
|
| 268 |
+
const auto eq = entry.find('=');
|
| 269 |
+
if (eq == std::string::npos) continue;
|
| 270 |
+
const std::string name = entry.substr(0, eq);
|
| 271 |
+
bool keep = false;
|
| 272 |
+
for (const char *n : kExact) if (name == n) { keep = true; break; }
|
| 273 |
+
for (const char *pfx : kPrefixes) if (!keep && name.rfind(pfx, 0) == 0) { keep = true; break; }
|
| 274 |
+
if (keep) out.push_back(entry);
|
| 275 |
+
}
|
| 276 |
+
return out;
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
std::string ModelManager::resolve_executable(const std::string &bin) {
|
| 280 |
+
if (bin.find('/') != std::string::npos) return bin;
|
| 281 |
+
const char *path = std::getenv("PATH");
|
| 282 |
+
std::string search = path ? path : "/usr/local/bin:/usr/bin:/bin";
|
| 283 |
+
size_t start = 0;
|
| 284 |
+
while (start <= search.size()) {
|
| 285 |
+
const auto end = search.find(':', start);
|
| 286 |
+
const std::string dir = search.substr(start, end == std::string::npos ? std::string::npos : end - start);
|
| 287 |
+
if (!dir.empty()) {
|
| 288 |
+
const std::string candidate = dir + "/" + bin;
|
| 289 |
+
if (access(candidate.c_str(), X_OK) == 0) return candidate;
|
| 290 |
+
}
|
| 291 |
+
if (end == std::string::npos) break;
|
| 292 |
+
start = end + 1;
|
| 293 |
+
}
|
| 294 |
+
return "";
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
bool ModelManager::wait_until_ready(pid_t pid, int port, int timeout_sec) {
|
| 298 |
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec);
|
| 299 |
// Exponential backoff: fast detection when the worker comes up (or dies)
|
cpp/model_manager.h
CHANGED
|
@@ -4,6 +4,7 @@
|
|
| 4 |
#include <optional>
|
| 5 |
#include <string>
|
| 6 |
#include <utility>
|
|
|
|
| 7 |
|
| 8 |
#include "llm_manager_types.h"
|
| 9 |
|
|
@@ -40,6 +41,9 @@ private:
|
|
| 40 |
int allocate_port();
|
| 41 |
void finish_switch(bool ok);
|
| 42 |
pid_t spawn_worker(const std::string &model, int port);
|
|
|
|
|
|
|
|
|
|
| 43 |
bool wait_until_ready(pid_t pid, int port, int timeout_sec);
|
| 44 |
std::pair<int, std::string> http_get(int port, const std::string &target);
|
| 45 |
};
|
|
|
|
| 4 |
#include <optional>
|
| 5 |
#include <string>
|
| 6 |
#include <utility>
|
| 7 |
+
#include <vector>
|
| 8 |
|
| 9 |
#include "llm_manager_types.h"
|
| 10 |
|
|
|
|
| 41 |
int allocate_port();
|
| 42 |
void finish_switch(bool ok);
|
| 43 |
pid_t spawn_worker(const std::string &model, int port);
|
| 44 |
+
// Allowlisted copy of our environment for the worker (no manager secrets).
|
| 45 |
+
static std::vector<std::string> worker_environment();
|
| 46 |
+
static std::string resolve_executable(const std::string &bin);
|
| 47 |
bool wait_until_ready(pid_t pid, int port, int timeout_sec);
|
| 48 |
std::pair<int, std::string> http_get(int port, const std::string &target);
|
| 49 |
};
|
cpp/runtime_components.cpp
CHANGED
|
@@ -19,8 +19,12 @@ RateLimitDecision RateLimiterStore::allow(const std::string &api_key_id, int est
|
|
| 19 |
if (requests_per_minute_ <= 0 && estimated_tokens_per_minute_ <= 0) return {};
|
| 20 |
|
| 21 |
std::lock_guard<std::mutex> lock(mu_);
|
| 22 |
-
auto &bucket = buckets_[api_key_id];
|
| 23 |
const auto now = std::chrono::steady_clock::now();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
refill(bucket.request_tokens, bucket.last_request_refill, requests_per_minute_, now);
|
| 25 |
refill(bucket.estimated_tokens, bucket.last_estimated_refill, estimated_tokens_per_minute_, now);
|
| 26 |
|
|
@@ -47,6 +51,23 @@ RateLimitDecision RateLimiterStore::allow(const std::string &api_key_id, int est
|
|
| 47 |
return {};
|
| 48 |
}
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
void RateLimiterStore::refill(
|
| 51 |
double &tokens,
|
| 52 |
std::chrono::steady_clock::time_point &last_refill,
|
|
|
|
| 19 |
if (requests_per_minute_ <= 0 && estimated_tokens_per_minute_ <= 0) return {};
|
| 20 |
|
| 21 |
std::lock_guard<std::mutex> lock(mu_);
|
|
|
|
| 22 |
const auto now = std::chrono::steady_clock::now();
|
| 23 |
+
if (buckets_.size() >= kMaxBuckets && buckets_.find(api_key_id) == buckets_.end()) {
|
| 24 |
+
prune_idle(now);
|
| 25 |
+
}
|
| 26 |
+
auto &bucket = buckets_[api_key_id];
|
| 27 |
+
bucket.last_seen = now;
|
| 28 |
refill(bucket.request_tokens, bucket.last_request_refill, requests_per_minute_, now);
|
| 29 |
refill(bucket.estimated_tokens, bucket.last_estimated_refill, estimated_tokens_per_minute_, now);
|
| 30 |
|
|
|
|
| 51 |
return {};
|
| 52 |
}
|
| 53 |
|
| 54 |
+
void RateLimiterStore::prune_idle(std::chrono::steady_clock::time_point now) {
|
| 55 |
+
for (auto it = buckets_.begin(); it != buckets_.end();) {
|
| 56 |
+
if (now - it->second.last_seen > kIdleEvict) it = buckets_.erase(it);
|
| 57 |
+
else ++it;
|
| 58 |
+
}
|
| 59 |
+
// Still full of active keys: drop the oldest ones so the map stays bounded
|
| 60 |
+
// even under a flood of fresh sources. Those sources merely get a fresh
|
| 61 |
+
// (full) bucket, which is no worse than before they were seen.
|
| 62 |
+
while (buckets_.size() >= kMaxBuckets) {
|
| 63 |
+
auto oldest = buckets_.begin();
|
| 64 |
+
for (auto it = buckets_.begin(); it != buckets_.end(); ++it) {
|
| 65 |
+
if (it->second.last_seen < oldest->second.last_seen) oldest = it;
|
| 66 |
+
}
|
| 67 |
+
buckets_.erase(oldest);
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
void RateLimiterStore::refill(
|
| 72 |
double &tokens,
|
| 73 |
std::chrono::steady_clock::time_point &last_refill,
|
cpp/runtime_components.h
CHANGED
|
@@ -25,13 +25,22 @@ private:
|
|
| 25 |
double estimated_tokens = 0.0;
|
| 26 |
std::chrono::steady_clock::time_point last_request_refill{};
|
| 27 |
std::chrono::steady_clock::time_point last_estimated_refill{};
|
|
|
|
| 28 |
};
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
std::mutex mu_;
|
| 31 |
std::unordered_map<std::string, Bucket> buckets_;
|
| 32 |
int requests_per_minute_;
|
| 33 |
int estimated_tokens_per_minute_;
|
| 34 |
|
|
|
|
|
|
|
| 35 |
static void refill(
|
| 36 |
double &tokens,
|
| 37 |
std::chrono::steady_clock::time_point &last_refill,
|
|
|
|
| 25 |
double estimated_tokens = 0.0;
|
| 26 |
std::chrono::steady_clock::time_point last_request_refill{};
|
| 27 |
std::chrono::steady_clock::time_point last_estimated_refill{};
|
| 28 |
+
std::chrono::steady_clock::time_point last_seen{};
|
| 29 |
};
|
| 30 |
|
| 31 |
+
// Bound on distinct keys kept (per-IP limiting sees arbitrary many
|
| 32 |
+
// sources). Above this, buckets idle longer than kIdleEvict are dropped;
|
| 33 |
+
// an idle bucket is full anyway, so evicting it changes nothing.
|
| 34 |
+
static constexpr size_t kMaxBuckets = 10000;
|
| 35 |
+
static constexpr std::chrono::seconds kIdleEvict{120};
|
| 36 |
+
|
| 37 |
std::mutex mu_;
|
| 38 |
std::unordered_map<std::string, Bucket> buckets_;
|
| 39 |
int requests_per_minute_;
|
| 40 |
int estimated_tokens_per_minute_;
|
| 41 |
|
| 42 |
+
void prune_idle(std::chrono::steady_clock::time_point now);
|
| 43 |
+
|
| 44 |
static void refill(
|
| 45 |
double &tokens,
|
| 46 |
std::chrono::steady_clock::time_point &last_refill,
|
cpp/server.cpp
CHANGED
|
@@ -284,9 +284,10 @@ static bool is_allowed_model(const ModelsConfig &models, const std::string &mode
|
|
| 284 |
return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
|
| 285 |
}
|
| 286 |
|
| 287 |
-
// Resolves the client address used for per-source rate limiting.
|
| 288 |
-
//
|
| 289 |
-
//
|
|
|
|
| 290 |
static std::string client_address(
|
| 291 |
const http::request<http::string_body> &req,
|
| 292 |
const std::string &peer_address,
|
|
@@ -295,10 +296,17 @@ static std::string client_address(
|
|
| 295 |
const auto it = req.find("X-Forwarded-For");
|
| 296 |
if (it != req.end()) {
|
| 297 |
std::string value = it->value().to_string();
|
| 298 |
-
const auto comma = value.
|
| 299 |
-
if (comma != std::string::npos) value = value.substr(
|
| 300 |
value = trim_copy(value);
|
| 301 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
}
|
| 303 |
}
|
| 304 |
return peer_address;
|
|
@@ -432,14 +440,22 @@ http::response<http::string_body> handle_request(
|
|
| 432 |
}
|
| 433 |
|
| 434 |
if (path == "/health" && req.method() == http::verb::get) {
|
| 435 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
}
|
| 437 |
|
| 438 |
if (path == "/models" && req.method() == http::verb::get) {
|
|
|
|
| 439 |
return json_response(http::status::ok, manager.models_view());
|
| 440 |
}
|
| 441 |
|
| 442 |
if (path == "/queue/metrics" && req.method() == http::verb::get) {
|
|
|
|
| 443 |
http::response<http::string_body> res{http::status::ok, req.version()};
|
| 444 |
res.set(http::field::content_type, "text/plain; version=0.0.4; charset=utf-8");
|
| 445 |
res.set(http::field::server, "llm-manager");
|
|
@@ -676,9 +692,12 @@ http::response<http::string_body> handle_request(
|
|
| 676 |
" final_state=" + state_to_string(final_state) +
|
| 677 |
" upstream_status=" + std::to_string(result.status) +
|
| 678 |
" elapsed_ms=" + std::to_string(elapsed_ms));
|
|
|
|
|
|
|
| 679 |
if (result.status >= 400) {
|
| 680 |
log_line("request_id=" + request_id +
|
| 681 |
-
"
|
|
|
|
| 682 |
}
|
| 683 |
return res;
|
| 684 |
}
|
|
@@ -725,7 +744,8 @@ http::response<http::string_body> handle_request(
|
|
| 725 |
" elapsed_ms=" + std::to_string(elapsed_ms));
|
| 726 |
if (upstream.status >= 400) {
|
| 727 |
log_line("request_id=" + request_id +
|
| 728 |
-
"
|
|
|
|
| 729 |
}
|
| 730 |
return res;
|
| 731 |
}
|
|
@@ -749,9 +769,7 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
|
|
| 749 |
} session_guard{deps.active_sessions};
|
| 750 |
|
| 751 |
const ManagerConfig &config = deps.config;
|
| 752 |
-
|
| 753 |
-
// keep-alive connections and stalled senders must eventually release it.
|
| 754 |
-
set_socket_timeouts(socket.native_handle(), config.server.read_timeout_sec, 60);
|
| 755 |
|
| 756 |
std::string peer_address;
|
| 757 |
{
|
|
@@ -760,6 +778,17 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
|
|
| 760 |
if (!ec) peer_address = endpoint.address().to_string();
|
| 761 |
}
|
| 762 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 763 |
try {
|
| 764 |
beast::flat_buffer buffer;
|
| 765 |
for (;;) {
|
|
@@ -768,17 +797,58 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
|
|
| 768 |
http::request_parser<http::string_body> parser;
|
| 769 |
parser.header_limit(16 * 1024);
|
| 770 |
parser.body_limit(static_cast<uint64_t>(config.limits.max_request_body_bytes));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 771 |
beast::error_code read_ec;
|
| 772 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 773 |
if (read_ec == http::error::body_limit || read_ec == http::error::header_limit) {
|
| 774 |
-
http::
|
| 775 |
-
res.set(http::field::content_type, "application/json");
|
| 776 |
-
res.set(http::field::server, "llm-manager");
|
| 777 |
-
res.keep_alive(false);
|
| 778 |
-
res.body() = R"({"error":"Request too large"})";
|
| 779 |
-
res.prepare_payload();
|
| 780 |
-
beast::error_code write_ec;
|
| 781 |
-
http::write(socket, res, write_ec);
|
| 782 |
break;
|
| 783 |
}
|
| 784 |
if (read_ec) throw beast::system_error{read_ec};
|
|
@@ -797,6 +867,7 @@ void do_session(boost::asio::ip::tcp::socket socket, ServerDeps &deps) {
|
|
| 797 |
// would_block/try_again family, which is the SO_RCVTIMEO receive timeout
|
| 798 |
// firing on an idle keep-alive connection. Neither is a real error.
|
| 799 |
if (e.code() != http::error::end_of_stream &&
|
|
|
|
| 800 |
e.code() != boost::asio::error::would_block &&
|
| 801 |
e.code() != boost::asio::error::try_again &&
|
| 802 |
e.code() != boost::asio::error::timed_out)
|
|
|
|
| 284 |
return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
|
| 285 |
}
|
| 286 |
|
| 287 |
+
// Resolves the client address used for per-source rate limiting. Behind a
|
| 288 |
+
// trusted reverse proxy (HF Spaces) the proxy appends the real client to
|
| 289 |
+
// X-Forwarded-For, so the LAST hop is the one it vouches for; anything
|
| 290 |
+
// before it was supplied by the client and must not be trusted.
|
| 291 |
static std::string client_address(
|
| 292 |
const http::request<http::string_body> &req,
|
| 293 |
const std::string &peer_address,
|
|
|
|
| 296 |
const auto it = req.find("X-Forwarded-For");
|
| 297 |
if (it != req.end()) {
|
| 298 |
std::string value = it->value().to_string();
|
| 299 |
+
const auto comma = value.rfind(',');
|
| 300 |
+
if (comma != std::string::npos) value = value.substr(comma + 1);
|
| 301 |
value = trim_copy(value);
|
| 302 |
+
// Only accept something that looks like an address: digits, hex,
|
| 303 |
+
// dots and colons. Anything else falls back to the TCP peer so a
|
| 304 |
+
// client cannot pick an arbitrary bucket key.
|
| 305 |
+
const bool address_like = !value.empty() && value.size() <= 45 &&
|
| 306 |
+
std::all_of(value.begin(), value.end(), [](unsigned char c) {
|
| 307 |
+
return std::isxdigit(c) || c == '.' || c == ':';
|
| 308 |
+
});
|
| 309 |
+
if (address_like) return value;
|
| 310 |
}
|
| 311 |
}
|
| 312 |
return peer_address;
|
|
|
|
| 440 |
}
|
| 441 |
|
| 442 |
if (path == "/health" && req.method() == http::verb::get) {
|
| 443 |
+
// Public liveness only: which model is loaded, counters and
|
| 444 |
+
// internals are behind a key (/models, /queue/metrics).
|
| 445 |
+
const auto view = manager.models_view();
|
| 446 |
+
return json_response(http::status::ok, {
|
| 447 |
+
{"status", view.value("status", "unknown")},
|
| 448 |
+
{"switch_in_progress", view.value("switch_in_progress", false)}
|
| 449 |
+
});
|
| 450 |
}
|
| 451 |
|
| 452 |
if (path == "/models" && req.method() == http::verb::get) {
|
| 453 |
+
if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
|
| 454 |
return json_response(http::status::ok, manager.models_view());
|
| 455 |
}
|
| 456 |
|
| 457 |
if (path == "/queue/metrics" && req.method() == http::verb::get) {
|
| 458 |
+
if (auto auth_res = ensure_authenticated(Role::USER)) return *auth_res;
|
| 459 |
http::response<http::string_body> res{http::status::ok, req.version()};
|
| 460 |
res.set(http::field::content_type, "text/plain; version=0.0.4; charset=utf-8");
|
| 461 |
res.set(http::field::server, "llm-manager");
|
|
|
|
| 692 |
" final_state=" + state_to_string(final_state) +
|
| 693 |
" upstream_status=" + std::to_string(result.status) +
|
| 694 |
" elapsed_ms=" + std::to_string(elapsed_ms));
|
| 695 |
+
// Upstream error bodies can echo user content; status and size
|
| 696 |
+
// are enough unless body logging is explicitly on.
|
| 697 |
if (result.status >= 400) {
|
| 698 |
log_line("request_id=" + request_id +
|
| 699 |
+
" upstream_error_bytes=" + std::to_string(result.body.size()) +
|
| 700 |
+
(config.logging.log_request_bodies ? " upstream_error_body=" + truncate_body(result.body) : ""));
|
| 701 |
}
|
| 702 |
return res;
|
| 703 |
}
|
|
|
|
| 744 |
" elapsed_ms=" + std::to_string(elapsed_ms));
|
| 745 |
if (upstream.status >= 400) {
|
| 746 |
log_line("request_id=" + request_id +
|
| 747 |
+
" proxied_get_error_bytes=" + std::to_string(upstream.body.size()) +
|
| 748 |
+
(config.logging.log_request_bodies ? " proxied_get_error_body=" + truncate_body(upstream.body) : ""));
|
| 749 |
}
|
| 750 |
return res;
|
| 751 |
}
|
|
|
|
| 769 |
} session_guard{deps.active_sessions};
|
| 770 |
|
| 771 |
const ManagerConfig &config = deps.config;
|
| 772 |
+
using clock = std::chrono::steady_clock;
|
|
|
|
|
|
|
| 773 |
|
| 774 |
std::string peer_address;
|
| 775 |
{
|
|
|
|
| 778 |
if (!ec) peer_address = endpoint.address().to_string();
|
| 779 |
}
|
| 780 |
|
| 781 |
+
auto write_and_close = [&](http::status status, const char *body) {
|
| 782 |
+
http::response<http::string_body> res{status, 11};
|
| 783 |
+
res.set(http::field::content_type, "application/json");
|
| 784 |
+
res.set(http::field::server, "llm-manager");
|
| 785 |
+
res.keep_alive(false);
|
| 786 |
+
res.body() = body;
|
| 787 |
+
res.prepare_payload();
|
| 788 |
+
beast::error_code write_ec;
|
| 789 |
+
http::write(socket, res, write_ec);
|
| 790 |
+
};
|
| 791 |
+
|
| 792 |
try {
|
| 793 |
beast::flat_buffer buffer;
|
| 794 |
for (;;) {
|
|
|
|
| 797 |
http::request_parser<http::string_body> parser;
|
| 798 |
parser.header_limit(16 * 1024);
|
| 799 |
parser.body_limit(static_cast<uint64_t>(config.limits.max_request_body_bytes));
|
| 800 |
+
|
| 801 |
+
// Read in pieces so an absolute deadline can be enforced. Before
|
| 802 |
+
// the first byte only the idle timeout applies (keep-alive); from
|
| 803 |
+
// the first byte on, the header and then the whole message must
|
| 804 |
+
// complete within fixed budgets regardless of how slowly bytes
|
| 805 |
+
// arrive. SO_RCVTIMEO is re-armed each iteration to the smaller of
|
| 806 |
+
// the idle timeout and the time left on the deadline.
|
| 807 |
beast::error_code read_ec;
|
| 808 |
+
clock::time_point first_byte{};
|
| 809 |
+
bool deadline_exceeded = false;
|
| 810 |
+
bool idle_closed = false;
|
| 811 |
+
while (!parser.is_done()) {
|
| 812 |
+
const auto now = clock::now();
|
| 813 |
+
long wait_sec = config.server.read_timeout_sec;
|
| 814 |
+
if (first_byte == clock::time_point{} && (parser.got_some() || buffer.size() > 0)) {
|
| 815 |
+
first_byte = now;
|
| 816 |
+
}
|
| 817 |
+
if (first_byte != clock::time_point{}) {
|
| 818 |
+
const int budget = parser.is_header_done() ? config.server.body_timeout_sec
|
| 819 |
+
: config.server.header_timeout_sec;
|
| 820 |
+
const auto limit = first_byte + std::chrono::seconds(budget);
|
| 821 |
+
if (now >= limit) {
|
| 822 |
+
deadline_exceeded = true;
|
| 823 |
+
break;
|
| 824 |
+
}
|
| 825 |
+
const long remaining = std::chrono::duration_cast<std::chrono::seconds>(limit - now).count() + 1;
|
| 826 |
+
wait_sec = std::min(wait_sec, remaining);
|
| 827 |
+
}
|
| 828 |
+
set_socket_timeouts(socket.native_handle(), static_cast<int>(std::max(1L, wait_sec)), 60);
|
| 829 |
+
|
| 830 |
+
read_ec = {};
|
| 831 |
+
http::read_some(socket, buffer, parser, read_ec);
|
| 832 |
+
if (read_ec == boost::asio::error::would_block ||
|
| 833 |
+
read_ec == boost::asio::error::try_again ||
|
| 834 |
+
read_ec == boost::asio::error::timed_out) {
|
| 835 |
+
if (first_byte == clock::time_point{}) {
|
| 836 |
+
idle_closed = true; // nothing started; plain keep-alive expiry
|
| 837 |
+
break;
|
| 838 |
+
}
|
| 839 |
+
continue; // deadline is re-checked at the top of the loop
|
| 840 |
+
}
|
| 841 |
+
if (read_ec) break;
|
| 842 |
+
}
|
| 843 |
+
|
| 844 |
+
if (idle_closed) break;
|
| 845 |
+
if (deadline_exceeded) {
|
| 846 |
+
log_line("session: request read deadline exceeded peer=" + peer_address);
|
| 847 |
+
write_and_close(http::status::request_timeout, R"({"error":"Request read timed out"})");
|
| 848 |
+
break;
|
| 849 |
+
}
|
| 850 |
if (read_ec == http::error::body_limit || read_ec == http::error::header_limit) {
|
| 851 |
+
write_and_close(http::status::payload_too_large, R"({"error":"Request too large"})");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 852 |
break;
|
| 853 |
}
|
| 854 |
if (read_ec) throw beast::system_error{read_ec};
|
|
|
|
| 867 |
// would_block/try_again family, which is the SO_RCVTIMEO receive timeout
|
| 868 |
// firing on an idle keep-alive connection. Neither is a real error.
|
| 869 |
if (e.code() != http::error::end_of_stream &&
|
| 870 |
+
e.code() != http::error::partial_message &&
|
| 871 |
e.code() != boost::asio::error::would_block &&
|
| 872 |
e.code() != boost::asio::error::try_again &&
|
| 873 |
e.code() != boost::asio::error::timed_out)
|
python/app.py
CHANGED
|
@@ -22,6 +22,16 @@ from logger import get_logger
|
|
| 22 |
|
| 23 |
logger = get_logger(__name__)
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
app = FastAPI(
|
| 26 |
title="AGI Multi-Model API",
|
| 27 |
description="""
|
|
@@ -110,7 +120,7 @@ BASE_PORT = int(os.getenv("BASE_PORT", "8080"))
|
|
| 110 |
PRELOAD_MODELS = os.getenv("PRELOAD_MODELS", "").split(",") if os.getenv("PRELOAD_MODELS") else []
|
| 111 |
WEB_SEARCH_CACHE_TTL = int(os.getenv("WEB_SEARCH_CACHE_TTL", "3600")) # 1 hour
|
| 112 |
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "300")) # 5 minutes
|
| 113 |
-
LOG_REQUEST_BODY = os.getenv("LOG_REQUEST_BODY", "
|
| 114 |
LOG_REQUEST_BODY_MAX_CHARS = int(os.getenv("LOG_REQUEST_BODY_MAX_CHARS", "2000"))
|
| 115 |
CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "2048"))
|
| 116 |
|
|
|
|
| 22 |
|
| 23 |
logger = get_logger(__name__)
|
| 24 |
|
| 25 |
+
# This legacy app has NO authentication: /switch-model, /cache/clear and the
|
| 26 |
+
# chat routes are public, and web-chat forwards queries to an external search
|
| 27 |
+
# provider. It is kept for reference only and is not part of the Docker image.
|
| 28 |
+
# Refuse to start unless the operator explicitly acknowledges that.
|
| 29 |
+
if os.getenv("LEGACY_UNAUTHENTICATED_APP_ACK") != "1":
|
| 30 |
+
raise RuntimeError(
|
| 31 |
+
"python/app.py is the unauthenticated legacy service. Use the C++ llm-manager, "
|
| 32 |
+
"or set LEGACY_UNAUTHENTICATED_APP_ACK=1 to run this on a trusted network only."
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
app = FastAPI(
|
| 36 |
title="AGI Multi-Model API",
|
| 37 |
description="""
|
|
|
|
| 120 |
PRELOAD_MODELS = os.getenv("PRELOAD_MODELS", "").split(",") if os.getenv("PRELOAD_MODELS") else []
|
| 121 |
WEB_SEARCH_CACHE_TTL = int(os.getenv("WEB_SEARCH_CACHE_TTL", "3600")) # 1 hour
|
| 122 |
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "300")) # 5 minutes
|
| 123 |
+
LOG_REQUEST_BODY = os.getenv("LOG_REQUEST_BODY", "0") == "1" # prompts are user data; opt-in only
|
| 124 |
LOG_REQUEST_BODY_MAX_CHARS = int(os.getenv("LOG_REQUEST_BODY_MAX_CHARS", "2000"))
|
| 125 |
CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "2048"))
|
| 126 |
|