File size: 9,474 Bytes
35d65e4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 | #!/usr/bin/env python3
"""
test_client.py v2 β Tests VRAM weight caching in vulkan_matmul_server
Protocol v2 (32-byte header):
magic(4) M(4) N(4) K(4) type(4) a_only(4) key_lo(4) key_hi(4)
type=0 plain matmul (A+B β C)
type=1 cached matmul
a_only=0: send A+B (weight will be cached on server)
a_only=1: send A only (weight should already be in VRAM cache)
Response: magic(4) status(4) M(4) N(4) then C data
status=0 OK
status=2 NEED_WEIGHT (evicted, must retry with a_only=0)
"""
import socket, struct, numpy as np, time, sys
from collections import defaultdict
MAGIC = 0x564B4D54
STATUS_OK = 0
STATUS_NEED_W = 2
TYPE_PLAIN = 0
TYPE_CACHED = 1
HOST = "127.0.0.1"
PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8097
# βββ Client-side cache tracking βββββββββββββββββββββββββββββββ
# Maps weight_key β (K, N) for weights we believe are cached on server
_warm_keys = {}
# Monotonic key counter β avoids id(B) address-reuse collisions.
# Python may reuse the same memory address for different numpy arrays if the
# old array is freed before the new one is allocated (same shape/size).
# Using id(B) as a cache key then causes the server to see a "cache hit" for
# a completely different weight matrix β protocol desync ("bad magic").
_key_seq = 0
def _next_key():
global _key_seq
_key_seq += 1
return _key_seq
def _connect():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
return s
# Persistent connection for this test session
_sock = _connect()
def _recv_all(n):
buf = b''
while len(buf) < n:
chunk = _sock.recv(min(65536, n - len(buf)))
if not chunk:
raise ConnectionError("Server closed")
buf += chunk
return buf
def _send_all(data):
mv = memoryview(data)
sent = 0
while sent < len(mv):
n = _sock.send(mv[sent:])
if n == 0:
raise ConnectionError("Send failed")
sent += n
def vk_matmul_cached(M, N, K, A, B, weight_key):
"""
Cached matmul: C[M,N] = A[M,K] Γ B[K,N]
On cache hit: only sends A (28KB for M=1 gen)
On cache miss: sends A+B, server caches B in VRAM
"""
global _warm_keys
a_only = 1 if weight_key in _warm_keys else 0
key_lo = weight_key & 0xFFFFFFFF
key_hi = (weight_key >> 32) & 0xFFFFFFFF
hdr = struct.pack('<IIIIIIII', MAGIC, M, N, K, TYPE_CACHED, a_only, key_lo, key_hi)
_send_all(hdr)
_send_all(A.astype(np.float32).tobytes())
if not a_only:
_send_all(B.astype(np.float32).tobytes())
# Response
rhdr = _recv_all(16)
magic, status, rM, rN = struct.unpack('<IIII', rhdr)
assert magic == MAGIC
if status == STATUS_NEED_W:
# Server evicted our key β retry with full B
del _warm_keys[weight_key]
return vk_matmul_cached(M, N, K, A, B, weight_key)
assert status == STATUS_OK, f"status={status}"
data = _recv_all(M * N * 4)
C = np.frombuffer(data, dtype=np.float32).reshape(M, N)
# Mark weight as cached on server
if not a_only:
_warm_keys[weight_key] = (K, N)
return C
def vk_matmul_plain(M, N, K, A, B):
"""Plain (uncached) matmul for comparison."""
hdr = struct.pack('<IIIIIIII', MAGIC, M, N, K, TYPE_PLAIN, 0, 0, 0)
_send_all(hdr)
_send_all(A.astype(np.float32).tobytes())
_send_all(B.astype(np.float32).tobytes())
rhdr = _recv_all(16)
magic, status, rM, rN = struct.unpack('<IIII', rhdr)
assert magic == MAGIC and status == 0
return np.frombuffer(_recv_all(M*N*4), dtype=np.float32).reshape(M, N)
# βββ Tests ββββββββββββββββββββββββββββββββββββββββββββββββββββ
print(f"=== Vulkan Matmul Server v2 Test β VRAM Cache ===")
print(f"Connecting to {HOST}:{PORT}...\n")
print("ββ 1. Correctness (plain vs cached) ββ")
for M, N, K, label in [
(1, 256, 256, "tiny"),
(1, 1024, 1024, "1K gen"),
(128, 2048, 2048, "128 prefill"),
(1, 2048, 7168, "MoE expert"),
]:
A = np.random.randn(M, K).astype(np.float32)
B = np.random.randn(K, N).astype(np.float32)
key = _next_key()
C_np = A @ B
C_vk = vk_matmul_cached(M, N, K, A, B, key)
err = np.max(np.abs(C_vk - C_np))
print(f" {'β' if err < 0.01 else 'β'} [{M}Γ{N}Γ{K}] {label:20s} maxerr={err:.2e}")
print()
print("ββ 2. Cache speedup β simulating 120B generation tokens ββ")
print(" (Upload weight ONCE, then send only activations for each token)")
print()
shapes = [
(7168, 7168, "q_proj/k_proj/v_proj (attention)"),
(7168, 7168, "o_proj (attention)"),
(7168, 2048, "expert gate_proj (MoE FFN)"),
(7168, 2048, "expert up_proj (MoE FFN)"),
(2048, 7168, "expert down_proj (MoE FFN)"),
]
N_TOKENS = 20
for (K, N, label) in shapes:
B = np.random.randn(K, N).astype(np.float32)
key = _next_key() # stable unique key β no id() address-reuse collisions
# Remove from warm cache so first call uploads (key is always fresh here)
_warm_keys.pop(key, None)
cold_times = []
warm_times = []
for tok in range(N_TOKENS):
A = np.random.randn(1, K).astype(np.float32) # single token
t0 = time.time()
C = vk_matmul_cached(1, N, K, A, B, key)
elapsed = (time.time() - t0) * 1000
if tok == 0:
cold_times.append(elapsed) # first call = upload + compute
else:
warm_times.append(elapsed) # subsequent = activation only + compute
cold_avg = sum(cold_times)
warm_avg = sum(warm_times) / len(warm_times)
speedup = cold_avg / warm_avg if warm_avg > 0 else 0
wt_mb = K * N * 4 / 1e6
print(f" [{K}Γ{N}] {label}")
print(f" Cold (upload {wt_mb:.0f}MB weight): {cold_avg:.0f}ms")
print(f" Warm (send 4KB activation only): {warm_avg:.1f}ms β {speedup:.0f}Γ faster")
print()
print()
print("ββ 3. Throughput: tokens/sec with cached weights ββ")
# Simulate a full 120B MoE forward pass for one token
# Real 120B MoE: ~80 layers, each with 4 attn projections + 2 active experts Γ 3 projections
LAYERS = 4 # subset for timing
ATTN_PROJS = 4 # Q,K,V,O
EXPERTS = 2 # active experts per token
EXP_PROJS = 3 # gate, up, down
# Pre-create weight matrices (simulating model loaded in POWER8 RAM)
attn_weights = [np.random.randn(7168, 7168).astype(np.float32) for _ in range(ATTN_PROJS)]
exp_weights = [(np.random.randn(7168, 2048).astype(np.float32),
np.random.randn(7168, 2048).astype(np.float32),
np.random.randn(2048, 7168).astype(np.float32))
for _ in range(EXPERTS)]
# Warm up the cache (first forward pass)
print(" Warming up weight cache (first token upload)...")
t_warmup = time.time()
A_dummy = np.random.randn(1, 7168).astype(np.float32)
for layer in range(LAYERS):
for wi, W in enumerate(attn_weights):
key = (layer * 1000 + wi) & 0xFFFFFFFFFFFFFFFF
_warm_keys.pop(key, None)
vk_matmul_cached(1, 7168, 7168, A_dummy, W, key)
for ei, (Wg, Wu, Wd) in enumerate(exp_weights):
kg = (layer * 1000 + 100 + ei * 3) & 0xFFFFFFFFFFFFFFFF
ku = (layer * 1000 + 101 + ei * 3) & 0xFFFFFFFFFFFFFFFF
kd = (layer * 1000 + 102 + ei * 3) & 0xFFFFFFFFFFFFFFFF
for k, W, K, N in [(kg,Wg,7168,2048),(ku,Wu,7168,2048),(kd,Wd,2048,7168)]:
_warm_keys.pop(k, None)
vk_matmul_cached(1, N, K, np.random.randn(1,K).astype(np.float32), W, k)
warmup_ms = (time.time() - t_warmup) * 1000
print(f" Warmup done in {warmup_ms:.0f}ms\n")
# Now time N tokens with warm cache
N_TIME_TOKENS = 10
t_start = time.time()
for tok in range(N_TIME_TOKENS):
A = np.random.randn(1, 7168).astype(np.float32)
for layer in range(LAYERS):
for wi, W in enumerate(attn_weights):
key = (layer * 1000 + wi) & 0xFFFFFFFFFFFFFFFF
vk_matmul_cached(1, 7168, 7168, A, W, key)
for ei, (Wg, Wu, Wd) in enumerate(exp_weights):
kg = (layer * 1000 + 100 + ei * 3) & 0xFFFFFFFFFFFFFFFF
ku = (layer * 1000 + 101 + ei * 3) & 0xFFFFFFFFFFFFFFFF
kd = (layer * 1000 + 102 + ei * 3) & 0xFFFFFFFFFFFFFFFF
vk_matmul_cached(1, 2048, 7168, np.random.randn(1,7168).astype(np.float32), Wg, kg)
vk_matmul_cached(1, 2048, 7168, np.random.randn(1,7168).astype(np.float32), Wu, ku)
vk_matmul_cached(1, 7168, 2048, np.random.randn(1,2048).astype(np.float32), Wd, kd)
elapsed_s = time.time() - t_start
ms_per_tok = elapsed_s * 1000 / N_TIME_TOKENS
ops_per_tok = LAYERS * (ATTN_PROJS + EXPERTS * EXP_PROJS)
print(f" {N_TIME_TOKENS} tokens Γ {LAYERS} layers Γ "
f"({ATTN_PROJS} attn + {EXPERTS}Γ{EXP_PROJS} MoE) ops")
print(f" Time: {elapsed_s*1000:.0f}ms total = {ms_per_tok:.1f}ms/token")
print(f" Throughput: {1000/ms_per_tok:.2f} tokens/sec "
f"({ops_per_tok} matmuls/token, {ms_per_tok/ops_per_tok:.1f}ms each)")
print(f"\n Cache stats: {len(_warm_keys)} weights resident in VRAM")
print(f" (Extrapolated full 80-layer 120B: "
f"~{ms_per_tok * 80/LAYERS:.0f}ms/token = "
f"~{1000/(ms_per_tok * 80/LAYERS):.2f} t/s)")
_sock.close()
print("\nAll done.")
|