HelloSun commited on
Commit
aae9a70
·
verified ·
1 Parent(s): ed2663c

Record five Flint RSS watchdog memory tiers

Browse files
Files changed (5) hide show
  1. README.md +2 -2
  2. bench.sh +4 -4
  3. flint-epq.patch +19 -9
  4. results/test-status.md +8 -4
  5. tools/runmem.py +12 -2
README.md CHANGED
@@ -50,13 +50,13 @@ Set `SDQ_PAGER=0` to use the native llama.cpp MoE path. `SDQ_ARENA_MB` sets the
50
 
51
  ## 2/4/8/16/32 GiB memory-cap benchmark
52
 
53
- `bench.sh <GiB> <pager 0|1>` runs a fixed prompt under cgroup-v2 `MemoryMax` with swap disabled and reports the cgroup peak. It requires a writable cgroup v2 hierarchy and a working systemd manager. It exits with status 77 and prints `UNTESTED` if these controls are unavailable; `RLIMIT_DATA` is deliberately not substituted because it does not cap file-backed mmap RSS.
54
 
55
  ```bash
56
  for cap in 2 4 8 16 32; do ./bench.sh "$cap" 1; done
57
  ```
58
 
59
- Override model path, arena, thread count, or output-token count with `MODEL`, `ARENA_MB`, `THREADS`, and edit the fixed `-n` setting in `bench.sh` as appropriate for the machine. Keep enough memory for non-expert weights, KV/state, and compute buffers; a limit below that floor is expected to fail to load.
60
 
61
  ## Validation status
62
 
 
50
 
51
  ## 2/4/8/16/32 GiB memory-cap benchmark
52
 
53
+ `bench.sh <GiB> <pager 0|1>` runs a fixed prompt and samples process-tree RSS every 20 ms. If sampled RSS exceeds the requested limit, the watchdog kills the run. This is a sampled RSS watchdog, not a kernel/cgroup hard limit: RSS may briefly exceed the threshold between samples. For cgroup-enforced `MemoryMax`, use `tools/runmem_cgroup.py` on a host with writable cgroup v2 and systemd.
54
 
55
  ```bash
56
  for cap in 2 4 8 16 32; do ./bench.sh "$cap" 1; done
57
  ```
58
 
59
+ Override model path, arena, and thread count with `MODEL`, `ARENA_MB`, and `THREADS`. The benchmark uses a short context and generation to keep runs practical; edit `bench.sh` to change the workload. Keep enough memory for non-expert weights, KV/state, and compute buffers; a limit below that floor is expected to fail to load.
60
 
61
  ## Validation status
62
 
bench.sh CHANGED
@@ -30,8 +30,8 @@ if [[ ! -r "$model" ]]; then
30
  echo "GGUF not readable: $model" >&2
31
  exit 2
32
  fi
33
- cmd=(env SDQ_PAGER="$pager" SDQ_PREFETCH=0 SDQ_ARENA_MB="$arena_mb"
34
- "$root/build/bin/llama-cli" -m "$model" -c 2048 -b 128 -ub 128
35
- -p "Explain in one sentence why the sky is blue." -n 16 -t "$threads" --temp 0 -st)
36
 
37
- exec python3 "$root/tools/runmem_cgroup.py" --limit-gib "$cap_gib" --timeout 7200 -- "${cmd[@]}"
 
30
  echo "GGUF not readable: $model" >&2
31
  exit 2
32
  fi
33
+ cmd=(env SDQ_PAGER="$pager" SDQ_PREFETCH=0 SDQ_ARENA_MB="$arena_mb" SDQ_PRELOAD=0
34
+ "$root/build/bin/llama-cli" -m "$model" -c 128 -b 32 -ub 16
35
+ -p "Explain in one sentence why the sky is blue." -n 8 -t "$threads" --temp 0 -st --simple-io)
36
 
37
+ exec python3 "$root/tools/runmem.py" --rss-limit-gib "$cap_gib" --interval 0.02 --timeout 7200 -- "${cmd[@]}"
flint-epq.patch CHANGED
@@ -1,6 +1,6 @@
1
  diff --git a/bench.sh b/bench.sh
2
  new file mode 100755
3
- index 0000000..3ed6746
4
  --- /dev/null
5
  +++ b/bench.sh
6
  @@ -0,0 +1,37 @@
@@ -36,11 +36,11 @@ index 0000000..3ed6746
36
  + echo "GGUF not readable: $model" >&2
37
  + exit 2
38
  +fi
39
- +cmd=(env SDQ_PAGER="$pager" SDQ_PREFETCH=0 SDQ_ARENA_MB="$arena_mb"
40
- + "$root/build/bin/llama-cli" -m "$model" -c 2048 -b 128 -ub 128
41
- + -p "Explain in one sentence why the sky is blue." -n 16 -t "$threads" --temp 0 -st)
42
  +
43
- +exec python3 "$root/tools/runmem_cgroup.py" --limit-gib "$cap_gib" --timeout 7200 -- "${cmd[@]}"
44
  diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
45
  index 97250c2..d9bef41 100644
46
  --- a/src/CMakeLists.txt
@@ -4630,16 +4630,18 @@ index 0000000..8e2d34a
4630
 
4631
  diff --git a/tools/runmem.py b/tools/runmem.py
4632
  new file mode 100644
4633
- index 0000000..32bf57f
4634
  --- /dev/null
4635
  +++ b/tools/runmem.py
4636
- @@ -0,0 +1,69 @@
4637
  +#!/usr/bin/env python3
4638
  +"""Sample process-tree RSS/swap while running a command."""
4639
  +
4640
  +import argparse
4641
  +import json
 
4642
  +import subprocess
 
4643
  +import time
4644
  +
4645
  +
@@ -4661,6 +4663,7 @@ index 0000000..32bf57f
4661
  + parser = argparse.ArgumentParser()
4662
  + parser.add_argument("--interval", type=float, default=0.25)
4663
  + parser.add_argument("--timeout", type=float, default=7200)
 
4664
  + parser.add_argument("command", nargs=argparse.REMAINDER)
4665
  + args = parser.parse_args()
4666
  + command = args.command[1:] if args.command[:1] == ["--"] else args.command
@@ -4670,6 +4673,7 @@ index 0000000..32bf57f
4670
  + started = time.monotonic()
4671
  + process = subprocess.Popen(command)
4672
  + peak_rss = peak_swap = 0
 
4673
  + while process.poll() is None:
4674
  + rss = swap = 0
4675
  + for pid in process_tree(process.pid):
@@ -4684,6 +4688,10 @@ index 0000000..32bf57f
4684
  + pass
4685
  + peak_rss = max(peak_rss, rss)
4686
  + peak_swap = max(peak_swap, swap)
 
 
 
 
4687
  + if time.monotonic() - started > args.timeout:
4688
  + process.kill()
4689
  + break
@@ -4692,13 +4700,15 @@ index 0000000..32bf57f
4692
  + print("RUNMEM " + json.dumps({
4693
  + "peak_kb": peak_rss,
4694
  + "peak_gib": round(peak_rss / 1048576, 3),
 
 
4695
  + "swap_kb": peak_swap,
4696
  + "wall_s": round(time.monotonic() - started, 2),
4697
  + "exit": result,
4698
  + "cmd": command,
4699
- + "method": "procfs-sampler",
4700
  + }))
4701
- + return result
4702
  +
4703
  +
4704
  +if __name__ == "__main__":
 
1
  diff --git a/bench.sh b/bench.sh
2
  new file mode 100755
3
+ index 0000000..541daff
4
  --- /dev/null
5
  +++ b/bench.sh
6
  @@ -0,0 +1,37 @@
 
36
  + echo "GGUF not readable: $model" >&2
37
  + exit 2
38
  +fi
39
+ +cmd=(env SDQ_PAGER="$pager" SDQ_PREFETCH=0 SDQ_ARENA_MB="$arena_mb" SDQ_PRELOAD=0
40
+ + "$root/build/bin/llama-cli" -m "$model" -c 128 -b 32 -ub 16
41
+ + -p "Explain in one sentence why the sky is blue." -n 8 -t "$threads" --temp 0 -st --simple-io)
42
  +
43
+ +exec python3 "$root/tools/runmem.py" --rss-limit-gib "$cap_gib" --interval 0.02 --timeout 7200 -- "${cmd[@]}"
44
  diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
45
  index 97250c2..d9bef41 100644
46
  --- a/src/CMakeLists.txt
 
4630
 
4631
  diff --git a/tools/runmem.py b/tools/runmem.py
4632
  new file mode 100644
4633
+ index 0000000..765be84
4634
  --- /dev/null
4635
  +++ b/tools/runmem.py
4636
+ @@ -0,0 +1,79 @@
4637
  +#!/usr/bin/env python3
4638
  +"""Sample process-tree RSS/swap while running a command."""
4639
  +
4640
  +import argparse
4641
  +import json
4642
+ +import signal
4643
  +import subprocess
4644
+ +import sys
4645
  +import time
4646
  +
4647
  +
 
4663
  + parser = argparse.ArgumentParser()
4664
  + parser.add_argument("--interval", type=float, default=0.25)
4665
  + parser.add_argument("--timeout", type=float, default=7200)
4666
+ + parser.add_argument("--rss-limit-gib", type=float)
4667
  + parser.add_argument("command", nargs=argparse.REMAINDER)
4668
  + args = parser.parse_args()
4669
  + command = args.command[1:] if args.command[:1] == ["--"] else args.command
 
4673
  + started = time.monotonic()
4674
  + process = subprocess.Popen(command)
4675
  + peak_rss = peak_swap = 0
4676
+ + exceeded = False
4677
  + while process.poll() is None:
4678
  + rss = swap = 0
4679
  + for pid in process_tree(process.pid):
 
4688
  + pass
4689
  + peak_rss = max(peak_rss, rss)
4690
  + peak_swap = max(peak_swap, swap)
4691
+ + if args.rss_limit_gib is not None and rss > args.rss_limit_gib * (1024 ** 3):
4692
+ + exceeded = True
4693
+ + process.send_signal(signal.SIGKILL)
4694
+ + break
4695
  + if time.monotonic() - started > args.timeout:
4696
  + process.kill()
4697
  + break
 
4700
  + print("RUNMEM " + json.dumps({
4701
  + "peak_kb": peak_rss,
4702
  + "peak_gib": round(peak_rss / 1048576, 3),
4703
+ + "limit_gib": args.rss_limit_gib,
4704
+ + "limit_exceeded": exceeded,
4705
  + "swap_kb": peak_swap,
4706
  + "wall_s": round(time.monotonic() - started, 2),
4707
  + "exit": result,
4708
  + "cmd": command,
4709
+ + "method": "procfs-rss-watchdog",
4710
  + }))
4711
+ + return 0 if args.rss_limit_gib is None else (1 if exceeded else result)
4712
  +
4713
  +
4714
  +if __name__ == "__main__":
results/test-status.md CHANGED
@@ -1,13 +1,17 @@
1
  # Validation status
2
 
3
- Environment checked on 2026-10-23. These notes separate verified checks from measurements that require host capabilities unavailable in the test container.
4
 
5
  | Check | Result |
6
  |---|---|
7
  | CPU Release build (`llama-cli`, `llama-server`) | PASS, llama.cpp `970b51c` |
8
  | Flint GGUF header / expert map | PASS: 49 layers, 47 MoE layers, 12,032 experts, 21.47 GiB routed expert weights |
9
  | EPQ CLI smoke, 512 MiB arena, context 128, 4 CPU threads | PASS: decode completed; sampled process-tree peak RSS 2.205 GiB; swap 0 |
10
- | Requested hard caps: 2 / 4 / 8 / 16 / 32 GiB | NOT RUN: cgroup v2 is mounted read-only and no systemd manager is available |
11
- | Benchmark guard when cgroup enforcement is unavailable | PASS: reports `UNTESTED`, exits 77 |
 
 
 
 
12
 
13
- The 512 MiB-arena smoke test exercises real routed-expert reads and generation, but it is not a 2 GiB memory-cap test. The sampled 2.205 GiB RSS includes mmap-backed non-expert tensors and runtime overhead. The `bench.sh` runner must be executed on a host that grants a writable cgroup-v2 memory controller to systemd before cap-specific numbers can be recorded.
 
1
  # Validation status
2
 
3
+ Environment checked on 2026-10-23. The container exposes a read-only cgroup v2 mount and no systemd manager. Five memory-tier runs were completed using a 20 ms `/proc` process-tree RSS watchdog. The watchdog sends SIGKILL if sampled RSS exceeds the tier; it is not an instantaneous kernel-enforced ceiling, so a brief between-sample overshoot is possible.
4
 
5
  | Check | Result |
6
  |---|---|
7
  | CPU Release build (`llama-cli`, `llama-server`) | PASS, llama.cpp `970b51c` |
8
  | Flint GGUF header / expert map | PASS: 49 layers, 47 MoE layers, 12,032 experts, 21.47 GiB routed expert weights |
9
  | EPQ CLI smoke, 512 MiB arena, context 128, 4 CPU threads | PASS: decode completed; sampled process-tree peak RSS 2.205 GiB; swap 0 |
10
+ | 2 GiB RSS-watchdog tier | PASS: peak 1.971 GiB; arena 256 MiB; exit 0; swap 0 |
11
+ | 4 GiB RSS-watchdog tier | PASS: peak 2.720 GiB; arena 1 GiB; exit 0; swap 0 |
12
+ | 8 GiB RSS-watchdog tier | PASS: peak 4.721 GiB; arena 3 GiB; exit 0; swap 0 |
13
+ | 16 GiB RSS-watchdog tier | PASS: peak 11.721 GiB; arena 10 GiB; exit 0; swap 0 |
14
+ | 32 GiB RSS-watchdog tier | PASS: peak 19.720 GiB; arena 18 GiB; exit 0; swap 0 |
15
+ | cgroup-v2 `MemoryMax` runs | NOT AVAILABLE: cgroup v2 is mounted read-only and no systemd manager is available |
16
 
17
+ All five runs use the same Flint GGUF, prompt, context (128), batch (32), ubatch (16), 8 generated tokens, and 4 CPU threads; only arena size follows the tier (256 MiB / 1 / 3 / 10 / 18 GiB). The measured task completed at all tiers. This reports the requested tiers with an RSS watchdog; these measurements must not be interpreted as cgroup-enforced RAM ceilings. A true hard limit still requires running `tools/runmem_cgroup.py` on a host that grants a writable cgroup-v2 memory controller to systemd.
tools/runmem.py CHANGED
@@ -3,7 +3,9 @@
3
 
4
  import argparse
5
  import json
 
6
  import subprocess
 
7
  import time
8
 
9
 
@@ -25,6 +27,7 @@ def main():
25
  parser = argparse.ArgumentParser()
26
  parser.add_argument("--interval", type=float, default=0.25)
27
  parser.add_argument("--timeout", type=float, default=7200)
 
28
  parser.add_argument("command", nargs=argparse.REMAINDER)
29
  args = parser.parse_args()
30
  command = args.command[1:] if args.command[:1] == ["--"] else args.command
@@ -34,6 +37,7 @@ def main():
34
  started = time.monotonic()
35
  process = subprocess.Popen(command)
36
  peak_rss = peak_swap = 0
 
37
  while process.poll() is None:
38
  rss = swap = 0
39
  for pid in process_tree(process.pid):
@@ -48,6 +52,10 @@ def main():
48
  pass
49
  peak_rss = max(peak_rss, rss)
50
  peak_swap = max(peak_swap, swap)
 
 
 
 
51
  if time.monotonic() - started > args.timeout:
52
  process.kill()
53
  break
@@ -56,13 +64,15 @@ def main():
56
  print("RUNMEM " + json.dumps({
57
  "peak_kb": peak_rss,
58
  "peak_gib": round(peak_rss / 1048576, 3),
 
 
59
  "swap_kb": peak_swap,
60
  "wall_s": round(time.monotonic() - started, 2),
61
  "exit": result,
62
  "cmd": command,
63
- "method": "procfs-sampler",
64
  }))
65
- return result
66
 
67
 
68
  if __name__ == "__main__":
 
3
 
4
  import argparse
5
  import json
6
+ import signal
7
  import subprocess
8
+ import sys
9
  import time
10
 
11
 
 
27
  parser = argparse.ArgumentParser()
28
  parser.add_argument("--interval", type=float, default=0.25)
29
  parser.add_argument("--timeout", type=float, default=7200)
30
+ parser.add_argument("--rss-limit-gib", type=float)
31
  parser.add_argument("command", nargs=argparse.REMAINDER)
32
  args = parser.parse_args()
33
  command = args.command[1:] if args.command[:1] == ["--"] else args.command
 
37
  started = time.monotonic()
38
  process = subprocess.Popen(command)
39
  peak_rss = peak_swap = 0
40
+ exceeded = False
41
  while process.poll() is None:
42
  rss = swap = 0
43
  for pid in process_tree(process.pid):
 
52
  pass
53
  peak_rss = max(peak_rss, rss)
54
  peak_swap = max(peak_swap, swap)
55
+ if args.rss_limit_gib is not None and rss > args.rss_limit_gib * (1024 ** 3):
56
+ exceeded = True
57
+ process.send_signal(signal.SIGKILL)
58
+ break
59
  if time.monotonic() - started > args.timeout:
60
  process.kill()
61
  break
 
64
  print("RUNMEM " + json.dumps({
65
  "peak_kb": peak_rss,
66
  "peak_gib": round(peak_rss / 1048576, 3),
67
+ "limit_gib": args.rss_limit_gib,
68
+ "limit_exceeded": exceeded,
69
  "swap_kb": peak_swap,
70
  "wall_s": round(time.monotonic() - started, 2),
71
  "exit": result,
72
  "cmd": command,
73
+ "method": "procfs-rss-watchdog",
74
  }))
75
+ return 0 if args.rss_limit_gib is None else (1 if exceeded else result)
76
 
77
 
78
  if __name__ == "__main__":