File size: 3,042 Bytes
41a3f6b a988b59 41a3f6b a988b59 41a3f6b a988b59 41a3f6b f158a05 41a3f6b 1502a42 41a3f6b 718f2c4 41a3f6b a988b59 41a3f6b a988b59 41a3f6b 718f2c4 41a3f6b 718f2c4 41a3f6b 718f2c4 41a3f6b 718f2c4 41a3f6b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """Modal worker for Tiny Narrator's llama.cpp reader brain.
Deploy:
modal deploy modal_workers/reader_brain.py
The deployed web server exposes llama.cpp's OpenAI-compatible API. Set:
LLAMA_CPP_BASE_URL=https://<modal-app-url>/v1
LLAMA_CPP_MODEL=narrator-brain
LLAMA_CPP_TOKEN=<same value as Modal secret>
The server intentionally binds to 0.0.0.0 because Modal's web_server proxy
routes traffic to the container port.
"""
import os
from pathlib import Path
import subprocess
import shutil
import modal
APP_NAME = "tiny-narrator-reader-brain"
MODEL_REF = "nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M"
MODEL_ALIAS = "narrator-brain"
SERVER_PORT = 8080
CACHE_DIR = "/cache"
app = modal.App(APP_NAME)
model_cache = modal.Volume.from_name("tiny-narrator-reader-brain-cache", create_if_missing=True)
def _secret_names() -> list[modal.Secret]:
return [modal.Secret.from_name("tiny-narrator-reader-brain-token")]
def _find_llama_server() -> str:
candidates = [
shutil.which("llama-server"),
"/app/llama-server",
"/usr/local/bin/llama-server",
"/usr/bin/llama-server",
"/bin/llama-server",
"/llama-server",
]
for candidate in candidates:
if candidate and Path(candidate).exists():
return candidate
inspected = []
for directory in ("/app", "/usr/local/bin", "/usr/bin", "/bin"):
path = Path(directory)
if path.exists():
inspected.append(f"{directory}: {[item.name for item in path.glob('*llama*')]}")
raise FileNotFoundError(f"llama-server binary was not found. Inspected: {'; '.join(inspected)}")
reader_brain_image = (
modal.Image.from_registry(
"ghcr.io/ggml-org/llama.cpp:server-cuda12",
add_python="3.12",
)
.dockerfile_commands("ENTRYPOINT []")
.env(
{
"HF_HOME": f"{CACHE_DIR}/huggingface",
}
)
)
@app.function(
image=reader_brain_image,
gpu="T4",
volumes={CACHE_DIR: model_cache},
secrets=_secret_names(),
timeout=900,
scaledown_window=300,
max_containers=1,
)
@modal.concurrent(max_inputs=20)
@modal.web_server(SERVER_PORT, startup_timeout=600)
def reader_brain_server():
api_key = os.getenv("LLAMA_CPP_TOKEN", "")
command = [
_find_llama_server(),
"--host",
"0.0.0.0",
"--port",
str(SERVER_PORT),
"-hf",
MODEL_REF,
"--alias",
MODEL_ALIAS,
"--ctx-size",
"4096",
"--parallel",
"1",
"--reasoning",
"off",
"--n-gpu-layers",
"999",
]
if api_key:
command.extend(["--api-key", api_key])
display_command = command.copy()
if "--api-key" in display_command:
key_index = display_command.index("--api-key") + 1
if key_index < len(display_command):
display_command[key_index] = "***"
print(f"[tiny-narrator-reader-brain] starting: {' '.join(display_command)}", flush=True)
subprocess.Popen(command)
|