File size: 3,753 Bytes
30bd7c0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
"""Real data for the explainer pages: one MOD tick of primes (state before, real after, network's answer and its
confidence), the primes source and machine code, and the training mix. Writes page_data.json.

Usage: python make_page_data.py <run> <step> <out.json>
"""
import json
import os
import sys

import numpy as np
import torch
import torch.nn as nn  # noqa: F401
import torch.nn.functional as F  # noqa: F401

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
sys.argv, args = sys.argv[:1] + ["--name", "_page"], sys.argv[1:]
import tiny8 as C  # noqa: E402
import tinyc  # noqa: E402
import gen as D  # noqa: E402

src = open(os.path.join(HERE, "train.py")).read()
ns = dict(globals())
exec(src[src.index("def slot_features"):src.index("def loss_fn")], ns)
run, step, out = args[0], args[1], args[2]
st = torch.load(f"/dev/shm/neucpu/runs/{run}/step_{step}.pt", map_location="cpu", weights_only=False)
model = ns["Console"](st["args"]["d"], st["args"]["layers"]).cuda().eval()
model.load_state_dict(st["model"])

psrc = open(os.path.join(HERE, "progs", "primes.c")).read()
rom, asm = tinyc.compile_c(psrc)
s = C.boot(rom, ())
states = [s]
for _ in range(20000):
    if C.halted(s):
        break
    s = C.step(s)
    states.append(s)
# pick a MOD tick a little into the run, where the result is not zero (a real remainder)
pick = None
for t, x in enumerate(states[:-1]):
    pc = int(x[C.O_PC])
    b0 = int(x[C.O_ROM + pc])
    if C.OPS[b0 >> 3] == "MOD" and t > 600:
        r, s2 = b0 & 7, int(x[C.O_ROM + pc + 1]) & 7
        if x[C.O_R + s2] > 2 and x[C.O_R + r] % x[C.O_R + s2] != 0:
            pick = t
            break
x, y = states[pick], states[pick + 1]
with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
    lg = model(torch.from_numpy(x[None]).cuda()).float()
pr = torch.softmax(lg[0], -1).cpu().numpy()  # [205, 256]
net = pr.argmax(-1)
changed = [int(i) for i in np.where(x[:C.N_STATE] != y[:C.N_STATE])[0]]
pc = int(x[C.O_PC])
top = {}
for i in changed + [C.O_PC]:
    o = np.argsort(-pr[i])[:5]
    top[i] = [[int(v), round(float(pr[i, v]), 6)] for v in o]

# a random training program and a random machine, as examples of what it learned from
g = np.random.default_rng(7)
pg = D.ProgGen(g)
rsrc, rrom = pg.compiled()
rm = D.random_machine(g, rrom, 0.5)

lines = [ln for ln in psrc.splitlines() if ln.strip() and not ln.strip().startswith("//")]
res = {
    "model": {"run": run, "step": int(step), "params_M": round(sum(p.numel() for p in model.parameters()) / 1e6, 1)},
    "layout": {"O_R": C.O_R, "O_PC": C.O_PC, "O_SP": C.O_SP, "O_FL": C.O_FL, "O_OP": C.O_OP, "O_IP": C.O_IP, "O_OUT": C.O_OUT,
               "N_OUT": C.N_OUT, "O_RAM": C.O_RAM, "N_RAM": C.N_RAM, "N_STATE": C.N_STATE, "O_ROM": C.O_ROM, "N_ROM": C.N_ROM,
               "O_IN": C.O_IN, "N_IN": C.N_IN, "N_TOK": C.N_TOK},
    "primes_src": lines, "primes_rom_bytes": len(rom), "primes_asm": asm if isinstance(asm, list) else str(asm).splitlines(),
    "primes_steps": len(states) - 1,
    "tick": {"t": pick, "pc": pc, "instr": C.disasm(rom, pc), "before": x.tolist(), "after": y[:C.N_STATE].tolist(),
             "net": net.tolist(), "net_exact": bool((net == y[:C.N_STATE]).all()), "changed": changed, "top": top,
             "min_conf": round(float(pr[np.arange(C.N_STATE), net].min()), 6)},
    "next_instr": [C.disasm(rom, (pc + 2 * k) & 255) for k in range(-3, 4)],
    "random_prog": rsrc.splitlines(), "random_machine_instr": C.disasm(bytes(rm[C.O_ROM:C.O_ROM + C.N_ROM]), int(rm[C.O_PC])),
    "random_machine": rm.tolist(),
    "ops": C.OPS,
}
json.dump(res, open(out, "w"))
print("PAGE_DATA_OK tick", pick, res["tick"]["instr"], "changed", changed, "net_exact", res["tick"]["net_exact"], flush=True)