sync learned_computer: 4 files from /dev/shm/neurv
Browse files- neurv/runs/rv3_s2000_copy.pt +3 -0
- neurv/rv_regs.py +30 -0
- neurv/stop_watch.sh +6 -0
- neurv/train_rv.py +13 -1
neurv/runs/rv3_s2000_copy.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe8ef36c3497daef7c43e9711189162a3b6eb14c490efc4d28d639723e2f92f9
|
| 3 |
+
size 3071399157
|
neurv/rv_regs.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Show unchanged-register mistakes: input value, predicted value, true value, which register."""
|
| 2 |
+
import collections, os, sys
|
| 3 |
+
import numpy as np, torch, torch.nn as nn, torch.nn.functional as F
|
| 4 |
+
HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE)
|
| 5 |
+
import rv, gen_rv as G
|
| 6 |
+
N_REG, N_TOK = rv.N_REG, G.N_TOK
|
| 7 |
+
src = open(os.path.join(HERE, "train_rv.py")).read()
|
| 8 |
+
ns = {"np": np, "torch": torch, "nn": nn, "F": F, "N_REG": N_REG, "N_TOK": N_TOK}
|
| 9 |
+
exec(src[src.index("def slot_features"):src.index("def loss_fn")], ns)
|
| 10 |
+
st = torch.load(sys.argv[1], map_location="cpu", weights_only=False)
|
| 11 |
+
model = ns["RVNet"](st["args"]["d"], st["args"]["layers"]).cuda().eval(); model.load_state_dict(st["model"])
|
| 12 |
+
progs = G.load_programs(sys.argv[2])[:40]
|
| 13 |
+
g = np.random.default_rng(7); uc = rv.new_uc(); X, Y = [], []
|
| 14 |
+
for p in progs:
|
| 15 |
+
for s, n in G.program_samples(uc, p, g, 6):
|
| 16 |
+
X.append(s); Y.append(n)
|
| 17 |
+
X = np.stack(X).astype(np.int64); Y = np.stack(Y).astype(np.int64)
|
| 18 |
+
with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
|
| 19 |
+
by = model(torch.from_numpy(X).cuda()).argmax(-1)
|
| 20 |
+
P = (by.long() << torch.tensor([0, 8, 16, 24], device="cuda")).sum(-1).cpu().numpy()
|
| 21 |
+
bad = collections.Counter(); tot = collections.Counter(); ex = []
|
| 22 |
+
for i in range(len(X)):
|
| 23 |
+
for r in range(32):
|
| 24 |
+
if X[i, r] == Y[i, r]:
|
| 25 |
+
tot[r] += 1
|
| 26 |
+
if P[i, r] != Y[i, r]:
|
| 27 |
+
bad[r] += 1
|
| 28 |
+
if len(ex) < 12: ex.append((r, hex(X[i, r]), hex(P[i, r]), hex(Y[i, r])))
|
| 29 |
+
print("wrong unchanged copies per register:", {r: f"{bad[r]}/{tot[r]}" for r in range(32) if tot[r]})
|
| 30 |
+
for e in ex: print("x%d in %s pred %s true %s" % e)
|
neurv/stop_watch.sh
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# stop a watched learned-RISC-V run: its watchdog first (so it cannot relaunch), then the run. usage: stop_watch.sh <name>
|
| 3 |
+
N=$1
|
| 4 |
+
for p in $(pgrep -f "bash /dev/shm/neurv/watch_rv.sh $N "); do kill $p; done
|
| 5 |
+
bash /dev/shm/neurv/kill_rv.sh $N
|
| 6 |
+
echo "watchdogs left $(pgrep -fc "watch_rv.sh $N ")"
|
neurv/train_rv.py
CHANGED
|
@@ -80,6 +80,11 @@ class RVNet(nn.Module):
|
|
| 80 |
self.norm = nn.RMSNorm(d)
|
| 81 |
self.head = nn.Linear(d, 4 * 256, bias=False)
|
| 82 |
self.copy = nn.Parameter(torch.tensor(4.0)) # copy prior: each output byte starts leaning toward its input byte
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
self.ckpt_grad = ckpt_grad
|
| 84 |
|
| 85 |
def forward(self, s): # s [B, N_TOK] int64 holding uint32 values -> logits [B, N_TOK, 4, 256]
|
|
@@ -87,6 +92,13 @@ class RVNet(nn.Module):
|
|
| 87 |
for k in range(4):
|
| 88 |
x = x + self.byte[k]((s >> (8 * k)) & 255)
|
| 89 |
x = x + self.bits(((s[..., None] >> self.shift) & 1).to(x.dtype))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
for b in self.blocks:
|
| 91 |
x = torch.utils.checkpoint.checkpoint(b, x, use_reentrant=False) if (self.ckpt_grad and self.training) else b(x)
|
| 92 |
lg = self.head(self.norm(x)).view(s.shape[0], N_TOK, 4, 256)
|
|
@@ -158,7 +170,7 @@ if __name__ == "__main__":
|
|
| 158 |
elif a.init:
|
| 159 |
st = torch.load(a.init, map_location="cpu", weights_only=False)
|
| 160 |
miss, unexp = model.load_state_dict(st["model"], strict=False)
|
| 161 |
-
assert
|
| 162 |
say(event="init", path=a.init, from_step=st["step"], missing=list(miss))
|
| 163 |
loader = iter(torch.utils.data.DataLoader(G.Stream(a.programs, seed=rank + 97 * start + 1, prog_share=a.prog_share, hard=a.hard),
|
| 164 |
batch_size=a.batch, num_workers=a.workers, pin_memory=True, persistent_workers=True,
|
|
|
|
| 80 |
self.norm = nn.RMSNorm(d)
|
| 81 |
self.head = nn.Linear(d, 4 * 256, bias=False)
|
| 82 |
self.copy = nn.Parameter(torch.tensor(4.0)) # copy prior: each output byte starts leaning toward its input byte
|
| 83 |
+
# fetch unit (fixed wiring, as in a real CPU): the word at memory[PC], read from the network's OWN input state,
|
| 84 |
+
# is shown to every token. Decoding and executing it is still entirely the network's job.
|
| 85 |
+
self.fbyte = nn.ModuleList([nn.Embedding(256, d) for _ in range(4)])
|
| 86 |
+
self.fbits = nn.Linear(32, d, bias=False)
|
| 87 |
+
self.fvalid = nn.Embedding(2, d)
|
| 88 |
self.ckpt_grad = ckpt_grad
|
| 89 |
|
| 90 |
def forward(self, s): # s [B, N_TOK] int64 holding uint32 values -> logits [B, N_TOK, 4, 256]
|
|
|
|
| 92 |
for k in range(4):
|
| 93 |
x = x + self.byte[k]((s >> (8 * k)) & 255)
|
| 94 |
x = x + self.bits(((s[..., None] >> self.shift) & 1).to(x.dtype))
|
| 95 |
+
pc = s[:, 32]
|
| 96 |
+
valid = (pc < (N_TOK - N_REG) * 4) & (pc % 4 == 0)
|
| 97 |
+
ins = torch.gather(s, 1, (N_REG + (pc // 4).clamp(0, N_TOK - N_REG - 1))[:, None])[:, 0] * valid
|
| 98 |
+
f = self.fvalid(valid.long()) + self.fbits(((ins[:, None] >> self.shift) & 1).to(x.dtype))
|
| 99 |
+
for k in range(4):
|
| 100 |
+
f = f + self.fbyte[k]((ins >> (8 * k)) & 255)
|
| 101 |
+
x = x + f[:, None, :]
|
| 102 |
for b in self.blocks:
|
| 103 |
x = torch.utils.checkpoint.checkpoint(b, x, use_reentrant=False) if (self.ckpt_grad and self.training) else b(x)
|
| 104 |
lg = self.head(self.norm(x)).view(s.shape[0], N_TOK, 4, 256)
|
|
|
|
| 170 |
elif a.init:
|
| 171 |
st = torch.load(a.init, map_location="cpu", weights_only=False)
|
| 172 |
miss, unexp = model.load_state_dict(st["model"], strict=False)
|
| 173 |
+
assert all(k == "copy" or k.startswith(("fbyte", "fbits", "fvalid")) for k in miss) and not unexp, (miss, unexp)
|
| 174 |
say(event="init", path=a.init, from_step=st["step"], missing=list(miss))
|
| 175 |
loader = iter(torch.utils.data.DataLoader(G.Stream(a.programs, seed=rank + 97 * start + 1, prog_share=a.prog_share, hard=a.hard),
|
| 176 |
batch_size=a.batch, num_workers=a.workers, pin_memory=True, persistent_workers=True,
|