File size: 3,466 Bytes
26f1c69 f48fa07 26f1c69 be7f527 f48fa07 26f1c69 f48fa07 26f1c69 f042cb3 26f1c69 f48fa07 26f1c69 e023532 26f1c69 e023532 26f1c69 e023532 26f1c69 e023532 26f1c69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | #!/usr/bin/env bash
# Janus-35B — create a small-context local tag for OpenAI /v1 clients.
#
# Ollama's /v1/chat/completions (OpenAI-compatible) silently ignores the
# `options` field, so it loads at the baked default `num_ctx 262144` — 19.77 GiB
# of weights plus a 262K f16 KV cache, ~25.4 GiB in total (~23.0 GiB with
# OLLAMA_KV_CACHE_TYPE=q8_0), which is too much for a tight host. Figures
# measured 2026-09-18; see the README's Hardware requirements. This bakes a
# local tag with num_ctx overridden to a loadable value (and num_batch 256),
# which any OpenAI /v1 client can point at.
# Reuses this repo's Modelfile verbatim (TEMPLATE / SYSTEM / stop params), only
# swapping the FROM line to the bundled blob and the num_ctx parameter.
#
# Usage:
# ./scripts/cap_ctx.sh # tag `janus`, num_ctx 4096
# CTX=8192 TAG=janus-cap ./scripts/cap_ctx.sh
# GGUF=/path/to/other.gguf ./scripts/cap_ctx.sh
#
# Requires: ollama, awk.
set -euo pipefail
CTX="${CTX:-4096}"
NBATCH="${NBATCH:-256}"
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
MODELFILE="${ROOT}/Modelfile"
GGUF="${GGUF:-${ROOT}/Janus-35B-A3B.Q4_K_M.gguf}"
TAG="${TAG:-janus}"
if ! command -v ollama >/dev/null 2>&1; then echo "[!] ollama not found in PATH" >&2; exit 1; fi
if [[ ! -f "${MODELFILE}" ]]; then echo "[!] missing ${MODELFILE}" >&2; exit 1; fi
# A fresh clone leaves ${GGUF} on disk as a ~136-byte git-LFS POINTER, not the
# weights. `[[ -f ]]` is true for that pointer, so the old existence check never
# fired and `ollama create` was handed a text file. Check the GGUF magic instead.
if [[ ! -f "${GGUF}" ]] || [[ "$(head -c 4 "${GGUF}" 2>/dev/null)" != "GGUF" ]]; then
echo "[!] ${GGUF} is not a usable GGUF." >&2
if [[ -f "${GGUF}" ]]; then
echo " It looks like an un-smudged git-LFS pointer." >&2
fi
echo " Fetch the weights with 'git lfs pull', or run './scripts/load_bundle.sh' - which" >&2
echo " downloads into .cache/ and prints the path - then rerun with GGUF=<that path>." >&2
exit 1
fi
echo "[*] tag: ${TAG}"
echo "[*] gguf: ${GGUF}"
echo "[*] num_ctx: ${CTX} num_batch: ${NBATCH}"
TMP="$(mktemp -t janus35b-capctx.XXXXXX)"
trap 'rm -f "${TMP}"' EXIT
# The END rule is load-bearing: the num_ctx rule only REWRITES an existing
# PARAMETER num_ctx line, so if the Modelfile ever loses it this script used to
# emit no override at all and still print "[+] Done. baked at num_ctx=..." - the
# exact OOM it exists to prevent, reported as success. live_check.sh has carried
# the same END fallback since 0.6.4.
awk -v g="${GGUF}" -v c="${CTX}" -v b="${NBATCH}" '
/^FROM[[:space:]]/ && !done { print "FROM " g; done=1; next }
/^PARAMETER[[:space:]]+num_ctx[[:space:]]/ { print "PARAMETER num_ctx " c; print "PARAMETER num_batch " b; n=1; next }
{ print }
END { if (!n) { print "PARAMETER num_ctx " c; print "PARAMETER num_batch " b } }
' "${MODELFILE}" > "${TMP}"
if ! grep -q '^PARAMETER[[:space:]]\+num_ctx[[:space:]]' "${TMP}"; then
echo "[!] internal error: patched Modelfile carries no num_ctx override" >&2
exit 1
fi
echo "[*] ollama create ${TAG} -f <patched modelfile>"
ollama create "${TAG}" -f "${TMP}"
echo
echo "[+] Done. ${TAG} is baked at num_ctx=${CTX} — usable from any OpenAI /v1 client:"
echo " curl http://localhost:11434/v1/chat/completions \\"
echo " -d '{\"model\":\"${TAG}\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}'"
|