Spaces:
Sleeping
Sleeping
Dmitry Beresnev commited on
Commit ·
6dc320d
1
Parent(s): 6471a06
fix other security volnurabilities
Browse files- Dockerfile +4 -2
- README.md +3 -1
- config.toml.example +2 -0
- cpp/config.cpp +1 -0
- cpp/llm_manager_types.h +3 -0
- cpp/runtime_components.cpp +3 -1
- cpp/runtime_components.h +1 -0
- cpp/server.cpp +68 -5
Dockerfile
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
|
| 3 |
# Install build dependencies
|
| 4 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
@@ -73,7 +74,7 @@ RUN g++ -std=c++17 -O2 -DBOOST_ERROR_CODE_HEADER_ONLY -pthread \
|
|
| 73 |
-o /build/llm-manager
|
| 74 |
|
| 75 |
# Runtime stage
|
| 76 |
-
FROM debian:bookworm-slim
|
| 77 |
|
| 78 |
# Install runtime dependencies including SSL/HTTPS support
|
| 79 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
@@ -148,6 +149,7 @@ EXPOSE 7860
|
|
| 148 |
# - ALLOWED_MODELS restricts /switch-model; the default model is always allowed.
|
| 149 |
# - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
|
| 150 |
# (X-Forwarded-For from the HF proxy).
|
|
|
|
| 151 |
ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
|
| 152 |
PER_IP_REQUESTS_PER_MINUTE=120 \
|
| 153 |
MANAGER_MAX_SESSIONS=256 \
|
|
|
|
| 1 |
+
# debian:bookworm-slim pinned by digest (resolved 2026-11-13). Bump deliberately.
|
| 2 |
+
FROM debian:bookworm-slim@sha256:734a72b3d6142a71993b4bc580ca01d8ca687f77a2c422d96a961d3376b179bc AS builder
|
| 3 |
|
| 4 |
# Install build dependencies
|
| 5 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
|
|
| 74 |
-o /build/llm-manager
|
| 75 |
|
| 76 |
# Runtime stage
|
| 77 |
+
FROM debian:bookworm-slim@sha256:734a72b3d6142a71993b4bc580ca01d8ca687f77a2c422d96a961d3376b179bc
|
| 78 |
|
| 79 |
# Install runtime dependencies including SSL/HTTPS support
|
| 80 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
|
|
| 149 |
# - ALLOWED_MODELS restricts /switch-model; the default model is always allowed.
|
| 150 |
# - PER_IP_REQUESTS_PER_MINUTE is a pre-auth limit per client address
|
| 151 |
# (X-Forwarded-For from the HF proxy).
|
| 152 |
+
# - MIN_FREE_DISK_MB refuses a model switch when the cache volume is nearly full.
|
| 153 |
ENV ALLOWED_MODELS=QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m \
|
| 154 |
PER_IP_REQUESTS_PER_MINUTE=120 \
|
| 155 |
MANAGER_MAX_SESSIONS=256 \
|
README.md
CHANGED
|
@@ -54,7 +54,9 @@ Other security-relevant settings (all overridable by environment variable):
|
|
| 54 |
| `[logging] log_request_bodies` | `LOG_REQUEST_BODIES` | Prompts stay out of logs unless enabled |
|
| 55 |
|
| 56 |
Cancellation is soft: cancelling a running request closes only that request's upstream connection.
|
| 57 |
-
Only the admin `/stop` endpoint restarts the worker.
|
|
|
|
|
|
|
| 58 |
`docs/SECURITY_AUDIT_AND_FIX_PLAN.md`.
|
| 59 |
|
| 60 |
## Features
|
|
|
|
| 54 |
| `[logging] log_request_bodies` | `LOG_REQUEST_BODIES` | Prompts stay out of logs unless enabled |
|
| 55 |
|
| 56 |
Cancellation is soft: cancelling a running request closes only that request's upstream connection.
|
| 57 |
+
Only the admin `/stop` endpoint restarts the worker. To be able to cancel, send your own
|
| 58 |
+
`X-Request-Id` header (8–64 chars of `[A-Za-z0-9_-]`) with the chat request and call
|
| 59 |
+
`POST /requests/<that id>/cancel` with the same key; an id already in flight is refused with 409. The full audit and rationale is in
|
| 60 |
`docs/SECURITY_AUDIT_AND_FIX_PLAN.md`.
|
| 61 |
|
| 62 |
## Features
|
config.toml.example
CHANGED
|
@@ -15,6 +15,8 @@ host = "127.0.0.1"
|
|
| 15 |
bind_host = "127.0.0.1"
|
| 16 |
base_port = 8080
|
| 17 |
switch_timeout_sec = 300
|
|
|
|
|
|
|
| 18 |
|
| 19 |
[llama]
|
| 20 |
n_ctx = 8192
|
|
|
|
| 15 |
bind_host = "127.0.0.1"
|
| 16 |
base_port = 8080
|
| 17 |
switch_timeout_sec = 300
|
| 18 |
+
# Refuse /switch-model when the model cache volume has less free space (MiB).
|
| 19 |
+
min_free_disk_mb = 512
|
| 20 |
|
| 21 |
[llama]
|
| 22 |
n_ctx = 8192
|
cpp/config.cpp
CHANGED
|
@@ -315,6 +315,7 @@ ManagerConfig load_manager_config() {
|
|
| 315 |
cfg.worker.bind_host = get_env_or("WORKER_BIND_HOST", get_toml_string_or(toml, "worker", "bind_host", cfg.worker.bind_host));
|
| 316 |
cfg.worker.base_port = get_env_int_or("WORKER_BASE_PORT", get_toml_int_or(toml, "worker", "base_port", cfg.worker.base_port));
|
| 317 |
cfg.worker.switch_timeout_sec = get_env_int_or("SWITCH_TIMEOUT_SEC", get_toml_int_or(toml, "worker", "switch_timeout_sec", cfg.worker.switch_timeout_sec));
|
|
|
|
| 318 |
|
| 319 |
cfg.llama.n_ctx = get_env_int_or("MODEL_N_CTX", get_toml_int_or(toml, "llama", "n_ctx", cfg.llama.n_ctx));
|
| 320 |
cfg.llama.threads = get_env_int_or("MODEL_THREADS", get_toml_int_or(toml, "llama", "threads", cfg.llama.threads));
|
|
|
|
| 315 |
cfg.worker.bind_host = get_env_or("WORKER_BIND_HOST", get_toml_string_or(toml, "worker", "bind_host", cfg.worker.bind_host));
|
| 316 |
cfg.worker.base_port = get_env_int_or("WORKER_BASE_PORT", get_toml_int_or(toml, "worker", "base_port", cfg.worker.base_port));
|
| 317 |
cfg.worker.switch_timeout_sec = get_env_int_or("SWITCH_TIMEOUT_SEC", get_toml_int_or(toml, "worker", "switch_timeout_sec", cfg.worker.switch_timeout_sec));
|
| 318 |
+
cfg.worker.min_free_disk_mb = std::max(0, get_env_int_or("MIN_FREE_DISK_MB", get_toml_int_or(toml, "worker", "min_free_disk_mb", cfg.worker.min_free_disk_mb)));
|
| 319 |
|
| 320 |
cfg.llama.n_ctx = get_env_int_or("MODEL_N_CTX", get_toml_int_or(toml, "llama", "n_ctx", cfg.llama.n_ctx));
|
| 321 |
cfg.llama.threads = get_env_int_or("MODEL_THREADS", get_toml_int_or(toml, "llama", "threads", cfg.llama.threads));
|
cpp/llm_manager_types.h
CHANGED
|
@@ -36,6 +36,9 @@ struct WorkerConfig {
|
|
| 36 |
std::string bind_host = "127.0.0.1";
|
| 37 |
int base_port = 8080;
|
| 38 |
int switch_timeout_sec = 300;
|
|
|
|
|
|
|
|
|
|
| 39 |
};
|
| 40 |
|
| 41 |
struct LlamaConfig {
|
|
|
|
| 36 |
std::string bind_host = "127.0.0.1";
|
| 37 |
int base_port = 8080;
|
| 38 |
int switch_timeout_sec = 300;
|
| 39 |
+
// /switch-model is refused when the model cache volume has less than
|
| 40 |
+
// this much free space (0 = no check).
|
| 41 |
+
int min_free_disk_mb = 512;
|
| 42 |
};
|
| 43 |
|
| 44 |
struct LlamaConfig {
|
cpp/runtime_components.cpp
CHANGED
|
@@ -81,7 +81,9 @@ std::shared_ptr<RequestContext> RequestRegistry::create(
|
|
| 81 |
ctx->enqueue_time = ctx->created_at;
|
| 82 |
|
| 83 |
std::lock_guard<std::mutex> lock(mu_);
|
| 84 |
-
|
|
|
|
|
|
|
| 85 |
return ctx;
|
| 86 |
}
|
| 87 |
|
|
|
|
| 81 |
ctx->enqueue_time = ctx->created_at;
|
| 82 |
|
| 83 |
std::lock_guard<std::mutex> lock(mu_);
|
| 84 |
+
// Never overwrite an in-flight entry: a colliding (client-supplied) id
|
| 85 |
+
// would otherwise let one caller detach another's request.
|
| 86 |
+
if (!requests_.emplace(request_id, ctx).second) return nullptr;
|
| 87 |
return ctx;
|
| 88 |
}
|
| 89 |
|
cpp/runtime_components.h
CHANGED
|
@@ -41,6 +41,7 @@ private:
|
|
| 41 |
|
| 42 |
class RequestRegistry {
|
| 43 |
public:
|
|
|
|
| 44 |
std::shared_ptr<RequestContext> create(
|
| 45 |
const std::string &request_id,
|
| 46 |
const ApiKeyRecord &principal,
|
|
|
|
| 41 |
|
| 42 |
class RequestRegistry {
|
| 43 |
public:
|
| 44 |
+
// Returns nullptr if `request_id` is already registered.
|
| 45 |
std::shared_ptr<RequestContext> create(
|
| 46 |
const std::string &request_id,
|
| 47 |
const ApiKeyRecord &principal,
|
cpp/server.cpp
CHANGED
|
@@ -10,6 +10,10 @@
|
|
| 10 |
|
| 11 |
#include <algorithm>
|
| 12 |
#include <atomic>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
#include <regex>
|
| 14 |
#include <sstream>
|
| 15 |
#include <unordered_set>
|
|
@@ -67,11 +71,17 @@ static json completion_payload_to_chat_payload(const json &payload, const Limits
|
|
| 67 |
}
|
| 68 |
}
|
| 69 |
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
if (payload.contains("n_predict") && payload["n_predict"].is_number_integer()) {
|
| 72 |
-
normalized_max_tokens = payload["n_predict"]
|
| 73 |
} else if (payload.contains("max_tokens") && payload["max_tokens"].is_number_integer()) {
|
| 74 |
-
normalized_max_tokens = payload["max_tokens"]
|
| 75 |
}
|
| 76 |
if (normalized_max_tokens <= 0) {
|
| 77 |
normalized_max_tokens = limits.default_max_tokens;
|
|
@@ -240,6 +250,36 @@ static bool is_valid_model_identifier(const std::string &model) {
|
|
| 240 |
return model.find("..") == std::string::npos;
|
| 241 |
}
|
| 242 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
static bool is_allowed_model(const ModelsConfig &models, const std::string &model) {
|
| 244 |
return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
|
| 245 |
}
|
|
@@ -277,7 +317,18 @@ http::response<http::string_body> handle_request(
|
|
| 277 |
Scheduler &scheduler = deps.scheduler;
|
| 278 |
|
| 279 |
const auto start = std::chrono::steady_clock::now();
|
| 280 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
const std::string target = req.target().to_string();
|
| 282 |
const std::string method = req.method_string().to_string();
|
| 283 |
const auto query_pos = target.find('?');
|
|
@@ -298,7 +349,8 @@ http::response<http::string_body> handle_request(
|
|
| 298 |
const std::string path = normalized ? *normalized : std::string();
|
| 299 |
|
| 300 |
log_line("request_id=" + request_id + " method=" + method + " path=" + raw_path +
|
| 301 |
-
" body_bytes=" + std::to_string(req.body().size())
|
|
|
|
| 302 |
if (config.logging.log_request_bodies && !req.body().empty()) {
|
| 303 |
log_line("request_id=" + request_id + " body=" + truncate_body(req.body()));
|
| 304 |
}
|
|
@@ -425,6 +477,12 @@ http::response<http::string_body> handle_request(
|
|
| 425 |
{"allowed_models", config.models.allowed}});
|
| 426 |
}
|
| 427 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 428 |
std::string err;
|
| 429 |
bool ok = manager.switch_model(model, err);
|
| 430 |
if (!ok) {
|
|
@@ -530,6 +588,11 @@ http::response<http::string_body> handle_request(
|
|
| 530 |
|
| 531 |
const std::string upstream_request_body = payload.dump();
|
| 532 |
auto ctx = registry.create(request_id, *authenticated, *estimate, upstream_request_body);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
// Once this handler returns, nobody can act on the request id anymore
|
| 534 |
// (cancellation targets in-flight requests only), so drop the registry
|
| 535 |
// entry on every exit path to keep the map bounded. The scheduler keeps
|
|
|
|
| 10 |
|
| 11 |
#include <algorithm>
|
| 12 |
#include <atomic>
|
| 13 |
+
#include <cctype>
|
| 14 |
+
#include <cstdint>
|
| 15 |
+
#include <cstdlib>
|
| 16 |
+
#include <filesystem>
|
| 17 |
#include <regex>
|
| 18 |
#include <sstream>
|
| 19 |
#include <unordered_set>
|
|
|
|
| 71 |
}
|
| 72 |
}
|
| 73 |
|
| 74 |
+
// Read in 64 bits so an oversized value is preserved (and then rejected
|
| 75 |
+
// by estimate_chat_tokens) instead of being truncated into range.
|
| 76 |
+
int64_t normalized_max_tokens = limits.default_max_tokens;
|
| 77 |
+
auto read_int64 = [](const json &v) -> int64_t {
|
| 78 |
+
if (v.is_number_unsigned() && v.get<uint64_t>() > static_cast<uint64_t>(INT64_MAX)) return INT64_MAX;
|
| 79 |
+
return v.get<int64_t>();
|
| 80 |
+
};
|
| 81 |
if (payload.contains("n_predict") && payload["n_predict"].is_number_integer()) {
|
| 82 |
+
normalized_max_tokens = read_int64(payload["n_predict"]);
|
| 83 |
} else if (payload.contains("max_tokens") && payload["max_tokens"].is_number_integer()) {
|
| 84 |
+
normalized_max_tokens = read_int64(payload["max_tokens"]);
|
| 85 |
}
|
| 86 |
if (normalized_max_tokens <= 0) {
|
| 87 |
normalized_max_tokens = limits.default_max_tokens;
|
|
|
|
| 250 |
return model.find("..") == std::string::npos;
|
| 251 |
}
|
| 252 |
|
| 253 |
+
// Client-chosen request ids: opaque tokens only, so they are safe in logs,
|
| 254 |
+
// headers and the cancel URL.
|
| 255 |
+
static bool is_valid_client_request_id(const std::string &id) {
|
| 256 |
+
if (id.size() < 8 || id.size() > 64) return false;
|
| 257 |
+
return std::all_of(id.begin(), id.end(), [](unsigned char c) {
|
| 258 |
+
return std::isalnum(c) || c == '-' || c == '_';
|
| 259 |
+
});
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
// Refuses a switch when the model cache volume is nearly full: -hf downloads
|
| 263 |
+
// into LLAMA_CACHE and a failed download leaves partial files behind.
|
| 264 |
+
static bool cache_has_free_space(int64_t min_free_bytes, std::string &detail) {
|
| 265 |
+
if (min_free_bytes <= 0) return true;
|
| 266 |
+
const char *cache = std::getenv("LLAMA_CACHE");
|
| 267 |
+
std::string dir = cache && *cache ? cache : std::string();
|
| 268 |
+
if (dir.empty()) {
|
| 269 |
+
const char *home = std::getenv("HOME");
|
| 270 |
+
dir = std::string(home && *home ? home : "/") + "/.cache/llama.cpp";
|
| 271 |
+
}
|
| 272 |
+
std::error_code ec;
|
| 273 |
+
auto info = std::filesystem::space(dir, ec);
|
| 274 |
+
if (ec) {
|
| 275 |
+
// Unknown filesystem state: do not block the switch, but say so.
|
| 276 |
+
detail = "unknown";
|
| 277 |
+
return true;
|
| 278 |
+
}
|
| 279 |
+
detail = std::to_string(info.available / (1024 * 1024)) + " MiB free";
|
| 280 |
+
return static_cast<int64_t>(info.available) >= min_free_bytes;
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
static bool is_allowed_model(const ModelsConfig &models, const std::string &model) {
|
| 284 |
return std::find(models.allowed.begin(), models.allowed.end(), model) != models.allowed.end();
|
| 285 |
}
|
|
|
|
| 317 |
Scheduler &scheduler = deps.scheduler;
|
| 318 |
|
| 319 |
const auto start = std::chrono::steady_clock::now();
|
| 320 |
+
// Clients may pick their own id (X-Request-Id) so they can cancel an
|
| 321 |
+
// in-flight request; otherwise one is generated. Duplicates are rejected
|
| 322 |
+
// at registration so one key cannot hijack another's id.
|
| 323 |
+
std::string request_id = generate_request_id();
|
| 324 |
+
bool client_supplied_id = false;
|
| 325 |
+
if (const auto it = req.find("X-Request-Id"); it != req.end()) {
|
| 326 |
+
const std::string candidate = trim_copy(it->value().to_string());
|
| 327 |
+
if (is_valid_client_request_id(candidate)) {
|
| 328 |
+
request_id = candidate;
|
| 329 |
+
client_supplied_id = true;
|
| 330 |
+
}
|
| 331 |
+
}
|
| 332 |
const std::string target = req.target().to_string();
|
| 333 |
const std::string method = req.method_string().to_string();
|
| 334 |
const auto query_pos = target.find('?');
|
|
|
|
| 349 |
const std::string path = normalized ? *normalized : std::string();
|
| 350 |
|
| 351 |
log_line("request_id=" + request_id + " method=" + method + " path=" + raw_path +
|
| 352 |
+
" body_bytes=" + std::to_string(req.body().size()) +
|
| 353 |
+
(client_supplied_id ? " client_request_id=true" : ""));
|
| 354 |
if (config.logging.log_request_bodies && !req.body().empty()) {
|
| 355 |
log_line("request_id=" + request_id + " body=" + truncate_body(req.body()));
|
| 356 |
}
|
|
|
|
| 477 |
{"allowed_models", config.models.allowed}});
|
| 478 |
}
|
| 479 |
|
| 480 |
+
std::string space_detail;
|
| 481 |
+
if (!cache_has_free_space(static_cast<int64_t>(config.worker.min_free_disk_mb) * 1024 * 1024, space_detail)) {
|
| 482 |
+
return json_response(http::status::insufficient_storage,
|
| 483 |
+
{{"error", "Not enough free disk in model cache (" + space_detail + ")"}});
|
| 484 |
+
}
|
| 485 |
+
|
| 486 |
std::string err;
|
| 487 |
bool ok = manager.switch_model(model, err);
|
| 488 |
if (!ok) {
|
|
|
|
| 588 |
|
| 589 |
const std::string upstream_request_body = payload.dump();
|
| 590 |
auto ctx = registry.create(request_id, *authenticated, *estimate, upstream_request_body);
|
| 591 |
+
if (!ctx) {
|
| 592 |
+
// Only reachable with a client-supplied id that is already in
|
| 593 |
+
// flight (ours or another key's).
|
| 594 |
+
return json_response(http::status::conflict, {{"error", "Request id already in flight"}});
|
| 595 |
+
}
|
| 596 |
// Once this handler returns, nobody can act on the request id anymore
|
| 597 |
// (cancellation targets in-flight requests only), so drop the registry
|
| 598 |
// entry on every exit path to keep the map bounded. The scheduler keeps
|