File size: 2,263 Bytes
511ba50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#!/bin/bash
# One-shot experiment suite for Ling-3.0-tiny VQ3 vs standard quantizations
# Usage: run_experiments.sh  (from work root)
set -u
cd "$(dirname "$0")/../.."
BIN=./llama.cpp/build/bin
CORP=corpora/wikitext-2-raw/wiki.test.raw
R=ling-vq3/results
mkdir -p "$R"
T=64
CHUNKS_PPL=256
CHUNKS_KLD=128

declare -A MODELS=(
  [bf16]=models/Ling-3.0-tiny-bf16.gguf
  [q8_0]=models/Ling-3.0-tiny-Q8_0.gguf
  [q4_k_m]=models/Ling-3.0-tiny-Q4_K_M.gguf
  [base_q8dq6]=models/Ling-3.0-tiny-base-Q8dQ6.gguf
  [vq3]=models/Ling-3.0-tiny-VQ3.gguf
)

echo "=== [1/5] PPL (wikitext-2 test, $CHUNKS_PPL chunks) ==="
for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do
  m=${MODELS[$k]}
  echo "--- PPL $k ---"
  $BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_PPL -t $T -ngl 0 \
    > "$R/ppl_$k.log" 2>&1
  grep -E "Final estimate|error" "$R/ppl_$k.log" | tail -2
done

echo "=== [2/5] Baseline logits from BF16 (KLD reference, $CHUNKS_KLD chunks) ==="
$BIN/llama-perplexity -m models/Ling-3.0-tiny-bf16.gguf -f "$CORP" -c 512 --chunks $CHUNKS_KLD \
  -t $T -ngl 0 --save-all-logits "$R/logits_bf16.bin" > "$R/logits_bf16.log" 2>&1
grep -E "Final estimate|error" "$R/logits_bf16.log" | tail -2

echo "=== [3/5] KLD vs BF16 ==="
for k in q8_0 q4_k_m base_q8dq6 vq3; do
  m=${MODELS[$k]}
  echo "--- KLD $k ---"
  $BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_KLD -t $T -ngl 0 \
    --kl-divergence --kl-divergence-base "$R/logits_bf16.bin" > "$R/kld_$k.log" 2>&1
  grep -E "Final estimate|error" "$R/kld_$k.log" | tail -2
done

echo "=== [4/5] llama-bench pp512/tg128 ==="
$BIN/llama-bench -t $T -p 512 -n 128 -r 2 -mlo 0 \
  -m models/Ling-3.0-tiny-bf16.gguf \
  -m models/Ling-3.0-tiny-Q8_0.gguf \
  -m models/Ling-3.0-tiny-Q4_K_M.gguf \
  -m models/Ling-3.0-tiny-base-Q8dQ6.gguf \
  -m models/Ling-3.0-tiny-VQ3.gguf > "$R/bench.tsv" 2> "$R/bench.err"
cat "$R/bench.tsv"

echo "=== [5/5] Peak RSS via short generation ==="
for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do
  m=${MODELS[$k]}
  /usr/bin/time -v $BIN/llama-cli -m "$m" -p "Hello" -n 8 -t $T --no-warmup -ngl 0 --temp 0 \
    > /dev/null 2> "$R/rss_$k.txt"
  rss=$(grep "Maximum resident" "$R/rss_$k.txt" | awk '{print $6}')
  echo "peak RSS $k: $rss KB"
done

echo "=== ALL DONE ==="