# -*- coding: utf-8 -*- """ stripe1flash mountain —— 0.000001B 参数(<=1000) 的字符级语言模型。 手写前向 + 反向传播 + Adam,仅依赖 numpy。 结构: 2 字符上下文 -> embedding(相加) -> tanh 隐藏层 -> softmax(V) """ import math, random, json, os, time import numpy as np MODEL_NAME = "stripe1flash mountain" HERE = os.path.dirname(os.path.abspath(__file__)) CKPT = os.path.join(HERE, "stripe1flash_mountain.json") np.random.seed(42); random.seed(42) # ---------------- 语料 ---------------- text = open(os.path.join(HERE, "corpus.txt"), encoding="utf-8").read() chars = sorted(set(text)) V = len(chars) c2i = {c: i for i, c in enumerate(chars)} i2c = {i: c for i, c in enumerate(chars)} data = np.array([c2i[c] for c in text], dtype=np.int32) T = len(data) # ---------------- 结构: 2*6 + 22*12 + 22 + 24*22 + 24 = 982 ---------------- N_CTX, D, H = 2, 6, 22 IN = N_CTX * D rng = np.random.default_rng(42) emb = rng.normal(0, 0.1, (V, D)) w1 = rng.uniform(-1 / math.sqrt(IN), 1 / math.sqrt(IN), (H, IN)) b1 = np.zeros(H) w2 = rng.uniform(-1 / math.sqrt(H), 1 / math.sqrt(H), (V, H)) b2 = np.zeros(V) N_PARAMS = sum(a.size for a in (emb, w1, b1, w2, b2)) print(f"模型: {MODEL_NAME}") print(f"语料 {T} 字符 | 词表 V={V} | 参数量 {N_PARAMS} (= {N_PARAMS/1e9:.6f} B)") assert N_PARAMS <= 1000 # Adam 状态 M = {k: np.zeros_like(v) for k, v in dict(e=emb, w1=w1, b1=b1, w2=w2, b2=b2).items()} S = {k: np.zeros_like(v) for k, v in dict(e=emb, w1=w1, b1=b1, w2=w2, b2=b2).items()} # ---------------- 训练 ---------------- BATCH, STEPS, LR = 256, 8000, 0.05 b1a, b2a, eps = 0.9, 0.999, 1e-8 t0 = time.time() for step in range(1, STEPS + 1): pos = rng.integers(N_CTX, T, size=BATCH) i1, i2, tgt = data[pos - 2], data[pos - 1], data[pos] # ---- 前向 ---- x = np.hstack([emb[i1], emb[i2]]) # (B, IN) pre = x @ w1.T + b1 # (B, H) h = np.tanh(pre) logits = h @ w2.T + b2 # (B, V) logits -= logits.max(1, keepdims=True) p = np.exp(logits) p /= p.sum(1, keepdims=True) # ---- 损失 ---- loss = -np.log(p[np.arange(BATCH), tgt] + 1e-12).mean() # ---- 反向 ---- dlogits = p.copy() dlogits[np.arange(BATCH), tgt] -= 1.0 dlogits /= BATCH gw2 = dlogits.T @ h gb2 = dlogits.sum(0) dh = dlogits @ w2 dpre = dh * (1.0 - h * h) gw1 = dpre.T @ x gb1 = dpre.sum(0) dx = dpre @ w1 demb = np.zeros_like(emb) np.add.at(demb, i1, dx[:, :D]) # 同一个 token 在一批里出现多次, 梯度要累加 np.add.at(demb, i2, dx[:, D:]) # ---- Adam(原地更新,不加 L2)---- lr = LR * (0.5 * (1 + math.cos(math.pi * step / STEPS))) bc1, bc2 = 1 - b1a ** step, 1 - b2a ** step for key, P, G in (("e", emb, demb), ("w1", w1, gw1), ("b1", b1, gb1), ("w2", w2, gw2), ("b2", b2, gb2)): M[key] = b1a * M[key] + (1 - b1a) * G S[key] = b2a * S[key] + (1 - b2a) * G * G P -= lr * (M[key] / bc1) / (np.sqrt(S[key] / bc2) + eps) if step % 500 == 0 or step == 1: print(f"step {step:5d} loss {loss:.4f} lr {lr:.4f} ({time.time()-t0:.0f}s)") json.dump({"name": MODEL_NAME, "params": int(N_PARAMS), "vocab": "".join(chars), "config": {"n_ctx": N_CTX, "d": D, "h": H}, "weights": {"emb": emb.tolist(), "w1": w1.tolist(), "b1": b1.tolist(), "w2": w2.tolist(), "b2": b2.tolist()}}, open(CKPT, "w", encoding="utf-8"), ensure_ascii=False) print("已保存:", CKPT) # ---------------- 采样 ---------------- def generate(prompt="t", n=160, temperature=0.7, seed=None): r = random.Random(seed) out = prompt ids = [c2i[c] for c in (" " + prompt)[-2:]] for _ in range(n): x = np.concatenate([emb[ids[0]], emb[ids[1]]]) h = np.tanh(w1 @ x + b1) z = w2 @ h + b2 z = z / temperature z -= z.max() q = np.exp(z); q /= q.sum() acc, nxt = 0.0, V - 1 u = r.random() for i, qi in enumerate(q): acc += qi if u <= acc: nxt = i break out += i2c[nxt] ids = [ids[1], nxt] return out print(f"\n--- {MODEL_NAME} 生成样例 ---") for k, pr in enumerate(["t", "the cat ", "the sun ", "the dog "]): print(f"[{k+1}] {generate(pr, seed=100 + k)!r}")