config.toml.example

[server]
host = "0.0.0.0"
port = 7860

[worker]
default_model = "QuantFactory/Qwen2.5-7B-Instruct-GGUF:q4_k_m"
llama_server_bin = "/usr/local/bin/llama-server"
host = "127.0.0.1"
# Loopback only: the worker has no auth of its own. Only bind wider if the
# manager runs on a different host than the worker.
bind_host = "127.0.0.1"
base_port = 8080
switch_timeout_sec = 300

[llama]
n_ctx = 8192
# Set threads to the number of vCPUs actually available; oversubscribing
# slows compute-bound generation. Batch sizes feed the BLAS prompt-eval
# path — validate changes with tests/bench.py.
threads = 2
ngl = 0
batch = 256
ubatch = 128

[auth]
header = "Authorization"
scheme = "Bearer"

[limits]
default_max_tokens = 256
max_tokens_per_request = 2048
request_timeout_sec = 30

[queue]
max_size = 100
max_tokens = 20000
admin_quota = 3
retry_after_sec = 5

[scheduler]
max_concurrent = 1

[streaming]
enabled = false

[rate_limit]
requests_per_minute = 60
estimated_tokens_per_minute = 6000

[[api_keys]]
key_id = "admin-main"
secret = "change-me-admin"
role = "admin"
enabled = true

[[api_keys]]
key_id = "user-main"
secret = "change-me-user"
role = "user"
enabled = true