mjpsm's picture
Deploy Progression API (llama-cpp-python)
63d4e3a verified
Raw History Blame Contribute Delete
1.04 kB
import os
MODEL_REPO_ID = os.getenv(
"MODEL_REPO_ID",
"mjpsm/activity-progression-model-600-qwen3.5-0.8b-gguf",
)
MODEL_FILENAME = os.getenv(
"MODEL_FILENAME",
"activity-progression-model-600-qwen3.5-0.8b-Q8_0.gguf",
)
MODEL_VERSION = os.getenv("MODEL_VERSION", "v1")
QUANTIZATION = "Q8_0"
RUNTIME_NAME = "llama-cpp-python"
PROMPT_FORMAT = "raw_sft_completion"
# Optional. When unset, huggingface_hub falls back to the token stored by
# `hf auth login`, which is what local development uses.
HF_TOKEN = os.getenv("HF_TOKEN")
LLAMA_CONTEXT_SIZE = int(os.getenv("LLAMA_CONTEXT_SIZE", "2048"))
LLAMA_THREADS = int(os.getenv("LLAMA_THREADS", "2"))
# 0 = CPU only. Set to -1 to offload every layer (e.g. Metal on Apple Silicon).
LLAMA_GPU_LAYERS = int(os.getenv("LLAMA_GPU_LAYERS", "0"))
# Internal generation settings. These match the settings the model was
# evaluated with and are deliberately NOT part of the public request schema.
MAX_NEW_TOKENS = 450
TEMPERATURE = 0.0
TOP_P = 1.0
REPEAT_PENALTY = 1.08
SEED = 42