scripts/live_check.sh

#!/usr/bin/env bash
# Live end-to-end checks of this repo's templates against a real Ollama.
#
# Starts an ISOLATED Ollama server - its own port and its own model store, never
# ~/.ollama or /var/lib/ollama - imports the bundled GGUF with this repo's
# Modelfile, and checks what render tests cannot see:
#   1. Ollama selects the Go template, with thinking detected
#   2. a first-turn tool call comes back exactly once
#   3. a replayed tool result with JSON-string arguments gets answered
#   4. reasoning from a turn before the latest user message IS replayed
#   5. reasoning inside a live tool-call chain IS kept
#   6. every reasoning_effort value Ollama accepts returns 200, and "none"
#      switches thinking off
#   7. the effort tier reaches the prompt: high/max and low add their
#      instruction, medium and unset add none
#
# Usage:
#   ./scripts/live_check.sh                  # the bundled GGUF (needs the real file,
#                                            # not the git-lfs pointer)
#   ./scripts/live_check.sh /path/to/model.gguf
# Env: PORT (11435), NUM_CTX (4096), EFFORT (high), OLLAMA (binary),
#      WORK_DIR (<repo>/.cache/live-check): each run uses a fresh subdirectory of
#      it and deletes only that; KEEP=1 keeps the run directory.
#
# Needs free disk on WORK_DIR of about twice the GGUF - ~42 GB for the bundled
# 21.23 GB Q4_K_M (Ollama keeps the imported file plus a rewritten copy, and
# briefly more during the import), RAM to load it, and tens of minutes on CPU for
# a 35B-A3B. Exits non-zero if any check fails.
#
# The server runs on CPU only: GPU backends are switched off in its environment,
# and the run stops before importing if Ollama still reports a GPU. On a shared
# iGPU, GPU compute can take the desktop session down with it.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
OLLAMA="${OLLAMA:-ollama}"
PORT="${PORT:-11435}"
NUM_CTX="${NUM_CTX:-4096}"
EFFORT="${EFFORT:-high}"
WORK_DIR="${WORK_DIR:-${ROOT}/.cache/live-check}"
HOST="http://127.0.0.1:${PORT}"
BUNDLE="$(awk '/^FROM[[:space:]]/{print $2; exit}' "${ROOT}/Modelfile")"
GGUF="${1:-${ROOT}/${BUNDLE#./}}"
TAG="live-check-$(basename "${ROOT}" | tr '[:upper:]' '[:lower:]')"

red()   { printf '\033[31m%s\033[0m\n' "$*"; }
green() { printf '\033[32m%s\033[0m\n' "$*"; }
blue()  { printf '\033[34m%s\033[0m\n' "$*"; }

for bin in "${OLLAMA}" python3 curl realpath; do
    command -v "${bin}" >/dev/null 2>&1 || { red "[!] ${bin} not found"; exit 1; }
done
if [[ ! -f "${GGUF}" ]] || [[ "$(head -c 4 "${GGUF}")" != "GGUF" ]]; then
    red "[!] ${GGUF} is not a usable GGUF (a fresh clone has a git-lfs pointer here)."
    red "    Pass a real GGUF path, or fetch the weights with 'git lfs pull'."
    exit 1
fi
GGUF="$(realpath -- "${GGUF}")"   # the generated Modelfile lives elsewhere, so FROM must be absolute
if curl -s -m 2 "${HOST}/api/version" >/dev/null; then
    red "[!] something is already listening on ${HOST}; set PORT to a free port."
    exit 1
fi

mkdir -p "${WORK_DIR}"
RUN_DIR="$(mktemp -d "${WORK_DIR}/run.XXXXXX")"
SERVER_PID=""
cleanup() {
    if [[ -n "${SERVER_PID}" ]]; then
        pkill -P "${SERVER_PID}" 2>/dev/null || true
        kill "${SERVER_PID}" 2>/dev/null || true
        wait "${SERVER_PID}" 2>/dev/null || true
    fi
    if [[ "${KEEP:-0}" != "1" ]]; then
        rm -rf "${RUN_DIR}"
        rmdir "${WORK_DIR}" 2>/dev/null || true   # only if nothing else is in it
    fi
}
trap cleanup EXIT

blue "[*] isolated Ollama on ${HOST}, store ${RUN_DIR}/models"
OLLAMA_MODELS="${RUN_DIR}/models" OLLAMA_HOST="127.0.0.1:${PORT}" \
OLLAMA_LLM_LIBRARY=cpu OLLAMA_VULKAN=0 GGML_VK_VISIBLE_DEVICES=-1 \
CUDA_VISIBLE_DEVICES=-1 ROCR_VISIBLE_DEVICES=-1 HIP_VISIBLE_DEVICES=-1 \
    "${OLLAMA}" serve > "${RUN_DIR}/serve.log" 2>&1 &
SERVER_PID=$!
for _ in $(seq 1 60); do
    curl -s -m 2 "${HOST}/api/version" >/dev/null && break
    kill -0 "${SERVER_PID}" 2>/dev/null || break
    sleep 1
done
if ! curl -s -m 2 "${HOST}/api/version" >/dev/null; then
    red "[!] the isolated Ollama did not start; last lines of its log:"
    tail -20 "${RUN_DIR}/serve.log"
    exit 1
fi
# Every ollama and curl call below must reach the isolated server, never the
# user's own (default 127.0.0.1:11434).
export OLLAMA_HOST="127.0.0.1:${PORT}"
# Ollama logs one "inference compute" line per device it will use, after device
# discovery, which can finish after the API answers.
for _ in $(seq 1 30); do
    grep -q 'msg="inference compute"' "${RUN_DIR}/serve.log" && break
    sleep 1
done
if ! grep -q 'msg="inference compute"' "${RUN_DIR}/serve.log"; then
    red "[!] the isolated Ollama logged no inference device; cannot confirm CPU only."
    exit 1
fi
if awk '/msg="inference compute"/ && !/library=cpu/ { gpu = 1 } END { exit !gpu }' "${RUN_DIR}/serve.log"; then
    red "[!] the isolated Ollama still reports a GPU; refusing to run on it:"
    grep 'msg="inference compute"' "${RUN_DIR}/serve.log"
    exit 1
fi
blue "[*] CPU only: $(grep -c 'msg="inference compute".*library=cpu' "${RUN_DIR}/serve.log") CPU device(s), no GPU"

GGUF="${GGUF}" NUM_CTX="${NUM_CTX}" awk '
    /^FROM[[:space:]]/ && !d { print "FROM " ENVIRON["GGUF"]; d=1; next }
    /^PARAMETER num_ctx/     { print "PARAMETER num_ctx " ENVIRON["NUM_CTX"]; n=1; next }
    { print }
    END { if (!n) print "PARAMETER num_ctx " ENVIRON["NUM_CTX"] }' "${ROOT}/Modelfile" > "${RUN_DIR}/Modelfile"
blue "[*] importing ${GGUF} as ${TAG} (copies it into the isolated store)"
# Inline env prefix, not just the ambient export above: this is the one call that
# WRITES a model store, and 0.6.1-0.6.3 shipped with the export accidentally
# dropped, which imported the blob straight into the user's own store. The export
# stays (removing it is what broke 0.6.1) and the /api/show guard below stays, but
# that guard fires after create and so can only detect, never prevent.
if ! OLLAMA_MODELS="${RUN_DIR}/models" OLLAMA_HOST="127.0.0.1:${PORT}" \
     "${OLLAMA}" create "${TAG}" -f "${RUN_DIR}/Modelfile" > "${RUN_DIR}/create.log" 2>&1; then
    red "[!] ollama create failed:"; tail -20 "${RUN_DIR}/create.log"; exit 1
fi
if [[ "$(curl -s -o /dev/null -w '%{http_code}' "${HOST}/api/show" -d "{\"model\":\"${TAG}\"}")" != 200 ]]; then
    red "[!] ${TAG} is not on the isolated server ${HOST}; refusing to run checks elsewhere."
    exit 1
fi

PASSES=0
FAILS=0
pass() { green "  [ ok ] $*"; PASSES=$((PASSES + 1)); }
fail() { red "  [FAIL] $*"; FAILS=$((FAILS + 1)); }
TOOLS='[{"type":"function","function":{"name":"get_weather","description":"Get current weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]'
MARK='MARKER alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa quebec romeo sierra tango uniform victor whiskey xray yankee zulu'

v1() {  # POST /v1/chat/completions; body in $1; prints the HTTP code (000 if the request failed)
    rm -f "${RUN_DIR}/resp.json"
    curl -s -m 900 -o "${RUN_DIR}/resp.json" -w '%{http_code}' \
        "${HOST}/v1/chat/completions" -H 'Content-Type: application/json' -d "$1" || true
}
jq_py() {  # evaluate a python expression over the last response as d ("" if it doesn't apply)
    python3 -c "import json, sys
try:
    d = json.load(open(sys.argv[1])); print($1)
except Exception:
    print('')" "${RUN_DIR}/resp.json"
}
prompt_tokens() {  # /api/chat on a fresh load; prints prompt_eval_count ("" on error)
    { curl -s -m 900 "${HOST}/api/chat" -d "$1" || true; } | python3 -c 'import json, sys
try:
    print(json.load(sys.stdin).get("prompt_eval_count", ""))
except Exception:
    print("")'
}

blue "[*] 1-2. first-turn tool call (reasoning_effort=${EFFORT}) and template selection"
code=$(v1 "{\"model\":\"${TAG}\",\"messages\":[{\"role\":\"user\",\"content\":\"Call get_weather for Tokyo. Respond ONLY with the tool call.\"}],\"tools\":${TOOLS},\"reasoning_effort\":\"${EFFORT}\",\"max_tokens\":1500,\"temperature\":0}")
if [[ "${code}" == "200" ]]; then
    n=$(jq_py "len(d['choices'][0]['message'].get('tool_calls') or [])")
    args=$(jq_py "[t['function']['arguments'] for t in (d['choices'][0]['message'].get('tool_calls') or [])]")
    if [[ "${n}" == "1" && "${args}" == *Tokyo* ]]; then pass "exactly one tool call: ${args}"
    else fail "expected exactly one get_weather(Tokyo) call, got ${n}: ${args}"; fi
else
    fail "first-turn tool call returned HTTP ${code}"
fi
sel=$(grep -F 'msg="template selection"' "${RUN_DIR}/serve.log" | tail -1 || true)
if [[ "${sel}" == *selected=go_template* && "${sel}" == *'go_template="[completion tools thinking]"'* ]]; then
    pass "Ollama selected the Go template with thinking detected"
else
    fail "template selection: ${sel:-no selection line logged}"
fi

blue "[*] 3. replayed tool result with JSON-string arguments"
code=$(v1 "{\"model\":\"${TAG}\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Tokyo? Use the tool.\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Tokyo\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"c1\",\"content\":\"{\\\"temp_c\\\": 22, \\\"condition\\\": \\\"light rain\\\"}\"}],\"tools\":${TOOLS},\"reasoning_effort\":\"${EFFORT}\",\"max_tokens\":1000,\"temperature\":0}")
if [[ "${code}" == "200" ]]; then
    content=$(jq_py "(d['choices'][0]['message'].get('content') or '').strip()[:80]")
    fin=$(jq_py "d['choices'][0].get('finish_reason')")
    if [[ -n "${content}" ]]; then pass "answered (${fin}): ${content}"
    else fail "HTTP 200 but no answer (finish_reason ${fin}) - raise max_tokens if reasoning ran long"; fi
else
    fail "replay turn returned HTTP ${code}"
fi

blue "[*] 4-5. reasoning replay, same template, fresh load per request"
body() {  # $1 = extra assistant fields, $2 = message tail, $3 = tools field
    printf '{"model":"%s","keep_alive":0,"think":true,"stream":false%s,"options":{"num_predict":1},"messages":[%s]}' "${TAG}" "$3" "$2" | sed "s|__TH__|$1|"
}
prior='{"role":"user","content":"What is 17 times 23?"},{"role":"assistant","content":"391."__TH__},{"role":"user","content":"Now add 9."}'
chain='{"role":"user","content":"Weather in Tokyo?"},{"role":"assistant","content":""__TH__,"tool_calls":[{"function":{"name":"get_weather","arguments":{"city":"Tokyo"}}}]},{"role":"tool","content":"{\"temp_c\":22}"}'
a=$(prompt_tokens "$(body ",\"thinking\":\"${MARK}\"" "${prior}" "")")
b=$(prompt_tokens "$(body "" "${prior}" "")")
if [[ -n "${a}" && -n "${b}" && "${a}" -gt "${b}" ]]; then pass "prior turn's reasoning replayed (${a} > ${b} prompt tokens)"
else fail "prior turn's reasoning not replayed or unmeasured (${a} vs ${b})"; fi
c=$(prompt_tokens "$(body ",\"thinking\":\"${MARK}\"" "${chain}" ",\"tools\":${TOOLS}")")
d=$(prompt_tokens "$(body "" "${chain}" ",\"tools\":${TOOLS}")")
if [[ -n "${c}" && -n "${d}" && "${c}" -gt "${d}" ]]; then pass "live tool chain keeps its reasoning (${c} > ${d} prompt tokens)"
else fail "live tool chain lost its reasoning or unmeasured (${c} vs ${d})"; fi

blue "[*] 6. every reasoning_effort value Ollama accepts"
for e in minimal low medium high xhigh ultra max none; do
    code=$(v1 "{\"model\":\"${TAG}\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}],\"reasoning_effort\":\"${e}\",\"max_tokens\":1}")
    if [[ "${code}" == "200" ]]; then pass "reasoning_effort=${e} -> 200"; else fail "reasoning_effort=${e} -> HTTP ${code}"; fi
done
code=$(v1 "{\"model\":\"${TAG}\",\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in five words.\"}],\"reasoning_effort\":\"none\",\"max_tokens\":80,\"temperature\":0}")
r=$(jq_py "len(d['choices'][0]['message'].get('reasoning_content') or d['choices'][0]['message'].get('reasoning') or '')")
if [[ "${code}" == "200" && "${r}" == "0" ]]; then pass "reasoning_effort=none switches thinking off"
else fail "reasoning_effort=none: HTTP ${code}, ${r} chars of reasoning"; fi

blue "[*] 7. the effort tier reaches the prompt (prompt tokens, fresh load per request)"
eff() {  # $1 = think field or ""; prints prompt tokens for a one-line chat
    prompt_tokens "{\"model\":\"${TAG}\",\"keep_alive\":0,\"stream\":false${1},\"options\":{\"num_predict\":1},\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}"
}
unset_n=$(eff ""); med=$(eff ',"think":"medium"'); high=$(eff ',"think":"high"'); max=$(eff ',"think":"max"'); low=$(eff ',"think":"low"')
if [[ -n "${med}" && "${unset_n}" == "${med}" ]]; then pass "unset = medium, no instruction (${unset_n} = ${med} prompt tokens)"
else fail "unset and medium differ or unmeasured (${unset_n} vs ${med})"; fi
if [[ -n "${high}" && -n "${med}" && "${high}" -gt "${med}" && "${max}" == "${high}" ]]; then pass "high and max add the xhigh instruction (${high} = ${max} > ${med})"
else fail "high/max did not add the xhigh instruction (high ${high}, max ${max}, medium ${med})"; fi
if [[ -n "${low}" && -n "${med}" && "${low}" -gt "${med}" && "${low}" != "${high}" ]]; then pass "low adds the low instruction (${low} > ${med})"
else fail "low did not add its instruction (low ${low}, medium ${med}, high ${high})"; fi

echo
TOTAL=$((PASSES + FAILS))
if (( FAILS )); then red "[!] ${FAILS} of ${TOTAL} live checks failed - server log: ${RUN_DIR}/serve.log (rerun with KEEP=1 to keep it)"; exit 1; fi
green "[+] all ${TOTAL} live checks passed"