dew6's picture
Release stripe1flash mountain: 982-param char-level LM, trained from scratch
a9f0c10 verified
Raw History Blame Contribute Delete
4.41 kB
# -*- coding: utf-8 -*-
"""
stripe1flash mountain —— 0.000001B 参数(<=1000) 的字符级语言模型。
手写前向 + 反向传播 + Adam,仅依赖 numpy。
结构: 2 字符上下文 -> embedding(相加) -> tanh 隐藏层 -> softmax(V)
"""
import math, random, json, os, time
import numpy as np
MODEL_NAME = "stripe1flash mountain"
HERE = os.path.dirname(os.path.abspath(__file__))
CKPT = os.path.join(HERE, "stripe1flash_mountain.json")
np.random.seed(42); random.seed(42)
# ---------------- 语料 ----------------
text = open(os.path.join(HERE, "corpus.txt"), encoding="utf-8").read()
chars = sorted(set(text))
V = len(chars)
c2i = {c: i for i, c in enumerate(chars)}
i2c = {i: c for i, c in enumerate(chars)}
data = np.array([c2i[c] for c in text], dtype=np.int32)
T = len(data)
# ---------------- 结构: 2*6 + 22*12 + 22 + 24*22 + 24 = 982 ----------------
N_CTX, D, H = 2, 6, 22
IN = N_CTX * D
rng = np.random.default_rng(42)
emb = rng.normal(0, 0.1, (V, D))
w1 = rng.uniform(-1 / math.sqrt(IN), 1 / math.sqrt(IN), (H, IN))
b1 = np.zeros(H)
w2 = rng.uniform(-1 / math.sqrt(H), 1 / math.sqrt(H), (V, H))
b2 = np.zeros(V)
N_PARAMS = sum(a.size for a in (emb, w1, b1, w2, b2))
print(f"模型: {MODEL_NAME}")
print(f"语料 {T} 字符 | 词表 V={V} | 参数量 {N_PARAMS} (= {N_PARAMS/1e9:.6f} B)")
assert N_PARAMS <= 1000
# Adam 状态
M = {k: np.zeros_like(v) for k, v in dict(e=emb, w1=w1, b1=b1, w2=w2, b2=b2).items()}
S = {k: np.zeros_like(v) for k, v in dict(e=emb, w1=w1, b1=b1, w2=w2, b2=b2).items()}
# ---------------- 训练 ----------------
BATCH, STEPS, LR = 256, 8000, 0.05
b1a, b2a, eps = 0.9, 0.999, 1e-8
t0 = time.time()
for step in range(1, STEPS + 1):
pos = rng.integers(N_CTX, T, size=BATCH)
i1, i2, tgt = data[pos - 2], data[pos - 1], data[pos]
# ---- 前向 ----
x = np.hstack([emb[i1], emb[i2]]) # (B, IN)
pre = x @ w1.T + b1 # (B, H)
h = np.tanh(pre)
logits = h @ w2.T + b2 # (B, V)
logits -= logits.max(1, keepdims=True)
p = np.exp(logits)
p /= p.sum(1, keepdims=True)
# ---- 损失 ----
loss = -np.log(p[np.arange(BATCH), tgt] + 1e-12).mean()
# ---- 反向 ----
dlogits = p.copy()
dlogits[np.arange(BATCH), tgt] -= 1.0
dlogits /= BATCH
gw2 = dlogits.T @ h
gb2 = dlogits.sum(0)
dh = dlogits @ w2
dpre = dh * (1.0 - h * h)
gw1 = dpre.T @ x
gb1 = dpre.sum(0)
dx = dpre @ w1
demb = np.zeros_like(emb)
np.add.at(demb, i1, dx[:, :D]) # 同一个 token 在一批里出现多次, 梯度要累加
np.add.at(demb, i2, dx[:, D:])
# ---- Adam(原地更新,不加 L2)----
lr = LR * (0.5 * (1 + math.cos(math.pi * step / STEPS)))
bc1, bc2 = 1 - b1a ** step, 1 - b2a ** step
for key, P, G in (("e", emb, demb), ("w1", w1, gw1), ("b1", b1, gb1),
("w2", w2, gw2), ("b2", b2, gb2)):
M[key] = b1a * M[key] + (1 - b1a) * G
S[key] = b2a * S[key] + (1 - b2a) * G * G
P -= lr * (M[key] / bc1) / (np.sqrt(S[key] / bc2) + eps)
if step % 500 == 0 or step == 1:
print(f"step {step:5d} loss {loss:.4f} lr {lr:.4f} ({time.time()-t0:.0f}s)")
json.dump({"name": MODEL_NAME, "params": int(N_PARAMS), "vocab": "".join(chars),
"config": {"n_ctx": N_CTX, "d": D, "h": H},
"weights": {"emb": emb.tolist(), "w1": w1.tolist(), "b1": b1.tolist(),
"w2": w2.tolist(), "b2": b2.tolist()}},
open(CKPT, "w", encoding="utf-8"), ensure_ascii=False)
print("已保存:", CKPT)
# ---------------- 采样 ----------------
def generate(prompt="t", n=160, temperature=0.7, seed=None):
r = random.Random(seed)
out = prompt
ids = [c2i[c] for c in (" " + prompt)[-2:]]
for _ in range(n):
x = np.concatenate([emb[ids[0]], emb[ids[1]]])
h = np.tanh(w1 @ x + b1)
z = w2 @ h + b2
z = z / temperature
z -= z.max()
q = np.exp(z); q /= q.sum()
acc, nxt = 0.0, V - 1
u = r.random()
for i, qi in enumerate(q):
acc += qi
if u <= acc:
nxt = i
break
out += i2c[nxt]
ids = [ids[1], nxt]
return out
print(f"\n--- {MODEL_NAME} 生成样例 ---")
for k, pr in enumerate(["t", "the cat ", "the sun ", "the dog "]):
print(f"[{k+1}] {generate(pr, seed=100 + k)!r}")