Prahaladha commited on
Commit
f511ee6
·
verified ·
1 Parent(s): d4f88af

sync learned_computer: 4 files from /dev/shm/neurv

Browse files
neurv/runs/rv3_s2000_copy.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe8ef36c3497daef7c43e9711189162a3b6eb14c490efc4d28d639723e2f92f9
3
+ size 3071399157
neurv/rv_regs.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Show unchanged-register mistakes: input value, predicted value, true value, which register."""
2
+ import collections, os, sys
3
+ import numpy as np, torch, torch.nn as nn, torch.nn.functional as F
4
+ HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE)
5
+ import rv, gen_rv as G
6
+ N_REG, N_TOK = rv.N_REG, G.N_TOK
7
+ src = open(os.path.join(HERE, "train_rv.py")).read()
8
+ ns = {"np": np, "torch": torch, "nn": nn, "F": F, "N_REG": N_REG, "N_TOK": N_TOK}
9
+ exec(src[src.index("def slot_features"):src.index("def loss_fn")], ns)
10
+ st = torch.load(sys.argv[1], map_location="cpu", weights_only=False)
11
+ model = ns["RVNet"](st["args"]["d"], st["args"]["layers"]).cuda().eval(); model.load_state_dict(st["model"])
12
+ progs = G.load_programs(sys.argv[2])[:40]
13
+ g = np.random.default_rng(7); uc = rv.new_uc(); X, Y = [], []
14
+ for p in progs:
15
+ for s, n in G.program_samples(uc, p, g, 6):
16
+ X.append(s); Y.append(n)
17
+ X = np.stack(X).astype(np.int64); Y = np.stack(Y).astype(np.int64)
18
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
19
+ by = model(torch.from_numpy(X).cuda()).argmax(-1)
20
+ P = (by.long() << torch.tensor([0, 8, 16, 24], device="cuda")).sum(-1).cpu().numpy()
21
+ bad = collections.Counter(); tot = collections.Counter(); ex = []
22
+ for i in range(len(X)):
23
+ for r in range(32):
24
+ if X[i, r] == Y[i, r]:
25
+ tot[r] += 1
26
+ if P[i, r] != Y[i, r]:
27
+ bad[r] += 1
28
+ if len(ex) < 12: ex.append((r, hex(X[i, r]), hex(P[i, r]), hex(Y[i, r])))
29
+ print("wrong unchanged copies per register:", {r: f"{bad[r]}/{tot[r]}" for r in range(32) if tot[r]})
30
+ for e in ex: print("x%d in %s pred %s true %s" % e)
neurv/stop_watch.sh ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # stop a watched learned-RISC-V run: its watchdog first (so it cannot relaunch), then the run. usage: stop_watch.sh <name>
3
+ N=$1
4
+ for p in $(pgrep -f "bash /dev/shm/neurv/watch_rv.sh $N "); do kill $p; done
5
+ bash /dev/shm/neurv/kill_rv.sh $N
6
+ echo "watchdogs left $(pgrep -fc "watch_rv.sh $N ")"
neurv/train_rv.py CHANGED
@@ -80,6 +80,11 @@ class RVNet(nn.Module):
80
  self.norm = nn.RMSNorm(d)
81
  self.head = nn.Linear(d, 4 * 256, bias=False)
82
  self.copy = nn.Parameter(torch.tensor(4.0)) # copy prior: each output byte starts leaning toward its input byte
 
 
 
 
 
83
  self.ckpt_grad = ckpt_grad
84
 
85
  def forward(self, s): # s [B, N_TOK] int64 holding uint32 values -> logits [B, N_TOK, 4, 256]
@@ -87,6 +92,13 @@ class RVNet(nn.Module):
87
  for k in range(4):
88
  x = x + self.byte[k]((s >> (8 * k)) & 255)
89
  x = x + self.bits(((s[..., None] >> self.shift) & 1).to(x.dtype))
 
 
 
 
 
 
 
90
  for b in self.blocks:
91
  x = torch.utils.checkpoint.checkpoint(b, x, use_reentrant=False) if (self.ckpt_grad and self.training) else b(x)
92
  lg = self.head(self.norm(x)).view(s.shape[0], N_TOK, 4, 256)
@@ -158,7 +170,7 @@ if __name__ == "__main__":
158
  elif a.init:
159
  st = torch.load(a.init, map_location="cpu", weights_only=False)
160
  miss, unexp = model.load_state_dict(st["model"], strict=False)
161
- assert set(miss) <= {"copy"} and not unexp, (miss, unexp)
162
  say(event="init", path=a.init, from_step=st["step"], missing=list(miss))
163
  loader = iter(torch.utils.data.DataLoader(G.Stream(a.programs, seed=rank + 97 * start + 1, prog_share=a.prog_share, hard=a.hard),
164
  batch_size=a.batch, num_workers=a.workers, pin_memory=True, persistent_workers=True,
 
80
  self.norm = nn.RMSNorm(d)
81
  self.head = nn.Linear(d, 4 * 256, bias=False)
82
  self.copy = nn.Parameter(torch.tensor(4.0)) # copy prior: each output byte starts leaning toward its input byte
83
+ # fetch unit (fixed wiring, as in a real CPU): the word at memory[PC], read from the network's OWN input state,
84
+ # is shown to every token. Decoding and executing it is still entirely the network's job.
85
+ self.fbyte = nn.ModuleList([nn.Embedding(256, d) for _ in range(4)])
86
+ self.fbits = nn.Linear(32, d, bias=False)
87
+ self.fvalid = nn.Embedding(2, d)
88
  self.ckpt_grad = ckpt_grad
89
 
90
  def forward(self, s): # s [B, N_TOK] int64 holding uint32 values -> logits [B, N_TOK, 4, 256]
 
92
  for k in range(4):
93
  x = x + self.byte[k]((s >> (8 * k)) & 255)
94
  x = x + self.bits(((s[..., None] >> self.shift) & 1).to(x.dtype))
95
+ pc = s[:, 32]
96
+ valid = (pc < (N_TOK - N_REG) * 4) & (pc % 4 == 0)
97
+ ins = torch.gather(s, 1, (N_REG + (pc // 4).clamp(0, N_TOK - N_REG - 1))[:, None])[:, 0] * valid
98
+ f = self.fvalid(valid.long()) + self.fbits(((ins[:, None] >> self.shift) & 1).to(x.dtype))
99
+ for k in range(4):
100
+ f = f + self.fbyte[k]((ins >> (8 * k)) & 255)
101
+ x = x + f[:, None, :]
102
  for b in self.blocks:
103
  x = torch.utils.checkpoint.checkpoint(b, x, use_reentrant=False) if (self.ckpt_grad and self.training) else b(x)
104
  lg = self.head(self.norm(x)).view(s.shape[0], N_TOK, 4, 256)
 
170
  elif a.init:
171
  st = torch.load(a.init, map_location="cpu", weights_only=False)
172
  miss, unexp = model.load_state_dict(st["model"], strict=False)
173
+ assert all(k == "copy" or k.startswith(("fbyte", "fbits", "fvalid")) for k in miss) and not unexp, (miss, unexp)
174
  say(event="init", path=a.init, from_step=st["step"], missing=list(miss))
175
  loader = iter(torch.utils.data.DataLoader(G.Stream(a.programs, seed=rank + 97 * start + 1, prog_share=a.prog_share, hard=a.hard),
176
  batch_size=a.batch, num_workers=a.workers, pin_memory=True, persistent_workers=True,