File size: 4,228 Bytes
3fd1a35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | #!/usr/bin/env bash
# Run on the RK3588 board with two binaries and the same .l3r package.
# Keeps all logs; starts only one model at a time and cleans up its own server.
set -euo pipefail
if [[ $# != 3 ]]; then
echo "Usage: bash compare_service.sh BASELINE_BINARY CANDIDATE_BINARY MODEL.l3r" >&2
exit 2
fi
command -v jq >/dev/null
port=19093
if [[ -n $(ss -H -ltn "sport = :$port") ]]; then
echo "Test port $port is already occupied" >&2
exit 2
fi
results=$(mktemp -d /tmp/ling3-service-check.XXXXXX)
server_pid=
stop_server() {
if [[ -n $server_pid ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
server_pid=
fi
}
trap stop_server EXIT
export LING3_PREWARM_EXPERTS=1 LING3_EXPERT_BALANCED=1 LING3_EXPERT_ZERO_COPY=1
export LING3_GDN_CPU_PREFILL=1 LING3_GDN_CPU_FP32_STATE=1
engine_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
export LING3_GDN_PREFILL_DIR=${LING3_GDN_PREFILL_DIR:-$engine_root/gdn-prefill-t16}
unset LING3_PREFILL_W4A4 LING3_DISABLE_BATCH_INPUT_REUSE LING3_TRACE_BATCH
prompts=(
'你好，请用一句话介绍你自己。'
'我们正在测试一个完全本地运行的办公室助手。它需要理解中文、总结会议、整理待办事项，并且记住刚才的上下文。请给出三个简短的办公建议，不要使用复杂术语。'
'请继续参考前面的办公场景。假设今天上午召开产品会议，张经理负责确认需求，小李负责整理客户反馈，小王负责检查设备运行状态。下午需要汇总各自的进展，找出没有完成的事项，并为明天安排优先级。客户反馈包括语音识别偶尔不准确、回复速度需要提高、长时间使用后需要保持稳定。我们希望助手能够把问题分成已经解决、正在处理和需要补充信息三类。请简要说明应该如何组织这份报告，并给出一个最重要的注意事项。'
)
# Exercise more than one prefill block, even with the compact Chinese tokenizer.
prompts[2]+="补充要求：${prompts[2]}"
echo "results=$results"
for label in baseline candidate; do
binary=$1
[[ $label == candidate ]] && binary=$2
"$binary" serve "$3" "$port" >"$results/$label.server.log" 2>&1 &
server_pid=$!
ready=0
for ((attempt=0; attempt<60; ++attempt)); do
kill -0 "$server_pid" 2>/dev/null || { echo "$label server exited" >&2; exit 1; }
if curl -fsS --max-time 1 "http://127.0.0.1:$port/health" >"$results/$label.health.json" 2>/dev/null; then
ready=1
break
fi
sleep 1
done
[[ $ready == 1 ]] || { echo "$label readiness timed out" >&2; exit 1; }
curl -fsS --max-time 90 --data-binary '你好' \
"http://127.0.0.1:$port/v1/chat?max_tokens=4" >"$results/$label.warmup.jsonl"
curl -fsS -X POST "http://127.0.0.1:$port/v1/reset" >/dev/null
for turn in 0 1 2 3; do
index=$turn
if [[ $turn == 3 ]]; then
curl -fsS -X POST "http://127.0.0.1:$port/v1/reset" >/dev/null
index=0
fi
curl -fsS --max-time 90 -H 'Content-Type: text/plain; charset=utf-8' \
--data-binary "${prompts[index]}" \
"http://127.0.0.1:$port/v1/chat?max_tokens=16" >"$results/$label.$turn.jsonl"
jq -es 'any(.[]; .type == "finish" and .generated_tokens > 0)' \
"$results/$label.$turn.jsonl" >/dev/null
if [[ $turn == 2 ]]; then
jq -es 'any(.[]; .type == "finish" and .prompt_tokens > 128 and .prefill_batch_count >= 2)' \
"$results/$label.$turn.jsonl" >/dev/null
fi
jq -cs '[.[] | select(.type == "token") | .id]' \
"$results/$label.$turn.jsonl" >"$results/$label.$turn.ids"
jq -c 'select(.type == "finish") | {turn,position,prompt_tokens,prefill_batch_count,prefill_max_batch_rows,generated_tokens,ttft_ms}' \
"$results/$label.$turn.jsonl"
done
cmp "$results/$label.0.ids" "$results/$label.3.ids"
stop_server
done
for turn in 0 1 2 3; do
cmp "$results/baseline.$turn.ids" "$results/candidate.$turn.ids"
done
echo 'PASS: baseline/candidate token IDs match on all turns and after reset'
|