Update entrypoint.sh

entrypoint.sh

@@ -3,9 +3,13 @@ set -euo pipefail
 
 MODEL_PATH="${MODEL_DIR}/${MODEL_FILE}"
 
+# python3, not python: the CUDA base image is Ubuntu and ships no `python`
+# alias. Same reason the gateway is started with `python3 -m uvicorn` below.
 if [ ! -f "$MODEL_PATH" ]; then
   echo "[entrypoint] Downloading ${MODEL_FILE} from ${MODEL_REPO} ..."
-  python -c "
+  echo "[entrypoint] (several GB - enable persistent storage on the Space so"
+  echo "[entrypoint]  this does not repeat, on GPU time, at every cold start)"
+  python3 -c "
 from huggingface_hub import hf_hub_download
 hf_hub_download(repo_id='${MODEL_REPO}', filename='${MODEL_FILE}', local_dir='${MODEL_DIR}')
 "
@@ -13,22 +17,45 @@ else
   echo "[entrypoint] Found cached model at ${MODEL_PATH}"
 fi
 
-echo "[entrypoint] Starting llama-server (threads=${THREADS}, parallel=${PARALLEL_SLOTS}, ctx=${CTX_SIZE}) ..."
+# Rosetta's own embedded template uses a Jinja filter llama.cpp's minja engine
+# does not implement, and llama-server refuses to start rather than degrade -
+# so that family gets the bundled, minja-compatible rewrite instead. Hy-MT2
+# uses the template inside its GGUF.
+# Same inference rule as app/config.py, so setting MODEL_REPO alone keeps the
+# server flags and the prompt builder agreed on the format.
+RESOLVED_FORMAT="${PROMPT_FORMAT:-}"
+if [ -z "$RESOLVED_FORMAT" ]; then
+  case "$(echo "$MODEL_REPO" | tr '[:upper:]' '[:lower:]')" in
+    *rosetta*) RESOLVED_FORMAT="rosetta" ;;
+    *)         RESOLVED_FORMAT="hy-mt2" ;;
+  esac
+fi
+
+TEMPLATE_ARGS=()
+case "$RESOLVED_FORMAT" in
+  rosetta) TEMPLATE_ARGS=(--chat-template-file /app/chat-template-rosetta.jinja) ;;
+esac
+
+echo "[entrypoint] Starting llama-server (gpu-layers=${GPU_LAYERS:-all}, threads=${THREADS}, parallel=${PARALLEL_SLOTS}, ctx=${CTX_SIZE}) ..."
 /app/llama-server \
   --model "$MODEL_PATH" \
   --host "${LLAMA_SERVER_HOST}" \
   --port "${LLAMA_SERVER_PORT}" \
+  --gpu-layers "${GPU_LAYERS:-all}" \
   --threads "${THREADS}" \
   --threads-batch "${THREADS}" \
   --ctx-size "${CTX_SIZE}" \
   --parallel "${PARALLEL_SLOTS}" \
   --cont-batching \
   --jinja \
+  "${TEMPLATE_ARGS[@]}" \
   --no-webui &
 LLAMA_PID=$!
 
+# Generous ceiling: on a T4 the first load also JIT-compiles the PTX shipped
+# for sm_75, which the driver then caches.
 echo "[entrypoint] Waiting for llama-server to become healthy ..."
-for _ in $(seq 1 150); do
+for _ in $(seq 1 300); do
   if curl -sf "http://${LLAMA_SERVER_HOST}:${LLAMA_SERVER_PORT}/health" >/dev/null 2>&1; then
     echo "[entrypoint] llama-server is ready."
     break
@@ -41,7 +68,7 @@ for _ in $(seq 1 150); do
 done
 
 echo "[entrypoint] Starting API gateway on port ${PORT} ..."
-uvicorn app.main:app --host 0.0.0.0 --port "${PORT}" &
+python3 -m uvicorn app.main:app --host 0.0.0.0 --port "${PORT}" &
 API_PID=$!
 
 wait -n "$LLAMA_PID" "$API_PID"