tools/compare_service.sh

#!/usr/bin/env bash
# Run on the RK3588 board with two binaries and the same .l3r package.
# Keeps all logs; starts only one model at a time and cleans up its own server.
set -euo pipefail
if [[ $# != 3 ]]; then
    echo "Usage: bash compare_service.sh BASELINE_BINARY CANDIDATE_BINARY MODEL.l3r" >&2
    exit 2
fi
command -v jq >/dev/null
port=19093
if [[ -n $(ss -H -ltn "sport = :$port") ]]; then
    echo "Test port $port is already occupied" >&2
    exit 2
fi
results=$(mktemp -d /tmp/ling3-service-check.XXXXXX)
server_pid=
stop_server() {
    if [[ -n $server_pid ]]; then
        kill "$server_pid" 2>/dev/null || true
        wait "$server_pid" 2>/dev/null || true
        server_pid=
    fi
}
trap stop_server EXIT
export LING3_PREWARM_EXPERTS=1 LING3_EXPERT_BALANCED=1 LING3_EXPERT_ZERO_COPY=1
export LING3_GDN_CPU_PREFILL=1 LING3_GDN_CPU_FP32_STATE=1
engine_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
export LING3_GDN_PREFILL_DIR=${LING3_GDN_PREFILL_DIR:-$engine_root/gdn-prefill-t16}
unset LING3_PREFILL_W4A4 LING3_DISABLE_BATCH_INPUT_REUSE LING3_TRACE_BATCH
prompts=(
    '你好，请用一句话介绍你自己。'
    '我们正在测试一个完全本地运行的办公室助手。它需要理解中文、总结会议、整理待办事项，并且记住刚才的上下文。请给出三个简短的办公建议，不要使用复杂术语。'
    '请继续参考前面的办公场景。假设今天上午召开产品会议，张经理负责确认需求，小李负责整理客户反馈，小王负责检查设备运行状态。下午需要汇总各自的进展，找出没有完成的事项，并为明天安排优先级。客户反馈包括语音识别偶尔不准确、回复速度需要提高、长时间使用后需要保持稳定。我们希望助手能够把问题分成已经解决、正在处理和需要补充信息三类。请简要说明应该如何组织这份报告，并给出一个最重要的注意事项。'
)
# Exercise more than one prefill block, even with the compact Chinese tokenizer.
prompts[2]+="补充要求：${prompts[2]}"
echo "results=$results"
for label in baseline candidate; do
    binary=$1
    [[ $label == candidate ]] && binary=$2
    "$binary" serve "$3" "$port" >"$results/$label.server.log" 2>&1 &
    server_pid=$!
    ready=0
    for ((attempt=0; attempt<60; ++attempt)); do
        kill -0 "$server_pid" 2>/dev/null || { echo "$label server exited" >&2; exit 1; }
        if curl -fsS --max-time 1 "http://127.0.0.1:$port/health" >"$results/$label.health.json" 2>/dev/null; then
            ready=1
            break
        fi
        sleep 1
    done
    [[ $ready == 1 ]] || { echo "$label readiness timed out" >&2; exit 1; }
    curl -fsS --max-time 90 --data-binary '你好' \
        "http://127.0.0.1:$port/v1/chat?max_tokens=4" >"$results/$label.warmup.jsonl"
    curl -fsS -X POST "http://127.0.0.1:$port/v1/reset" >/dev/null
    for turn in 0 1 2 3; do
        index=$turn
        if [[ $turn == 3 ]]; then
            curl -fsS -X POST "http://127.0.0.1:$port/v1/reset" >/dev/null
            index=0
        fi
        curl -fsS --max-time 90 -H 'Content-Type: text/plain; charset=utf-8' \
            --data-binary "${prompts[index]}" \
            "http://127.0.0.1:$port/v1/chat?max_tokens=16" >"$results/$label.$turn.jsonl"
        jq -es 'any(.[]; .type == "finish" and .generated_tokens > 0)' \
            "$results/$label.$turn.jsonl" >/dev/null
        if [[ $turn == 2 ]]; then
            jq -es 'any(.[]; .type == "finish" and .prompt_tokens > 128 and .prefill_batch_count >= 2)' \
                "$results/$label.$turn.jsonl" >/dev/null
        fi
        jq -cs '[.[] | select(.type == "token") | .id]' \
            "$results/$label.$turn.jsonl" >"$results/$label.$turn.ids"
        jq -c 'select(.type == "finish") | {turn,position,prompt_tokens,prefill_batch_count,prefill_max_batch_rows,generated_tokens,ttft_ms}' \
            "$results/$label.$turn.jsonl"
    done
    cmp "$results/$label.0.ids" "$results/$label.3.ids"
    stop_server
done
for turn in 0 1 2 3; do
    cmp "$results/baseline.$turn.ids" "$results/candidate.$turn.ids"
done
echo 'PASS: baseline/candidate token IDs match on all turns and after reset'