#!/bin/bash
# usage: [CI_PORT=port] ci.sh <hf_ckpt_dir> <tag> [seeds...]: convert to GGUF, serve on CPU, run the llama.cpp function call
# script greedy then once per seed with the server default sampling, one line per run with the per scenario results
set -e
CK=$1; TAG=$2; shift 2; SEEDS=${@:-1 2 3 4 5}
cd "$(dirname "$0")"; source env.sh
L=$LLAMA_CPP; D=${CI_DIR:-/tmp/small-test-ci}; G=$D/$TAG-f16.gguf; PORT=${CI_PORT:-8094}; mkdir -p $D
[ -f $G ] || python3 convert.py $CK --outfile $G --outtype f16 > $D/conv_$TAG.log 2>&1
for S in greedy $SEEDS; do
  [ $S = greedy ] && X="--temp 0" || X="--seed $S"
  $L/build/bin/llama-server -m $G --jinja -ngl 0 -c 8192 --port $PORT -t 12 $X > /dev/null 2>&1 & P=$!; sleep 4
  R=$(python3 $L/scripts/server-test-function-call.py --port $PORT 2>&1 | sed 's/\x1b\[[0-9;]*m//g')
  echo "$TAG $S $(echo "$R" | grep Results | sed 's/.*Results: //; s/ passed//') | $(echo "$R" | grep -E '✔ PASS|✘ FAIL' | awk '{print $2}' | tr '\n' ' ')"
  kill $P; wait $P 2>/dev/null || true
done