Download train.py from dew6/stripe1flash-mountain: direct link, hf CLI and curl.
- Browser
- Download file 4.41 kB
-
https://huggingface.co/dew6/stripe1flash-mountain/resolve/main/train.py
- Command line
-
hf download hf://dew6/stripe1flash-mountain/train.py
-
curl -L -o train.py https://huggingface.co/dew6/stripe1flash-mountain/resolve/main/train.py
4.41 kB
| # -*- coding: utf-8 -*- | |
| """ | |
| stripe1flash mountain —— 0.000001B 参数(<=1000) 的字符级语言模型。 | |
| 手写前向 + 反向传播 + Adam,仅依赖 numpy。 | |
| 结构: 2 字符上下文 -> embedding(相加) -> tanh 隐藏层 -> softmax(V) | |
| """ | |
| import math, random, json, os, time | |
| import numpy as np | |
| MODEL_NAME = "stripe1flash mountain" | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| CKPT = os.path.join(HERE, "stripe1flash_mountain.json") | |
| np.random.seed(42); random.seed(42) | |
| # ---------------- 语料 ---------------- | |
| text = open(os.path.join(HERE, "corpus.txt"), encoding="utf-8").read() | |
| chars = sorted(set(text)) | |
| V = len(chars) | |
| c2i = {c: i for i, c in enumerate(chars)} | |
| i2c = {i: c for i, c in enumerate(chars)} | |
| data = np.array([c2i[c] for c in text], dtype=np.int32) | |
| T = len(data) | |
| # ---------------- 结构: 2*6 + 22*12 + 22 + 24*22 + 24 = 982 ---------------- | |
| N_CTX, D, H = 2, 6, 22 | |
| IN = N_CTX * D | |
| rng = np.random.default_rng(42) | |
| emb = rng.normal(0, 0.1, (V, D)) | |
| w1 = rng.uniform(-1 / math.sqrt(IN), 1 / math.sqrt(IN), (H, IN)) | |
| b1 = np.zeros(H) | |
| w2 = rng.uniform(-1 / math.sqrt(H), 1 / math.sqrt(H), (V, H)) | |
| b2 = np.zeros(V) | |
| N_PARAMS = sum(a.size for a in (emb, w1, b1, w2, b2)) | |
| print(f"模型: {MODEL_NAME}") | |
| print(f"语料 {T} 字符 | 词表 V={V} | 参数量 {N_PARAMS} (= {N_PARAMS/1e9:.6f} B)") | |
| assert N_PARAMS <= 1000 | |
| # Adam 状态 | |
| M = {k: np.zeros_like(v) for k, v in dict(e=emb, w1=w1, b1=b1, w2=w2, b2=b2).items()} | |
| S = {k: np.zeros_like(v) for k, v in dict(e=emb, w1=w1, b1=b1, w2=w2, b2=b2).items()} | |
| # ---------------- 训练 ---------------- | |
| BATCH, STEPS, LR = 256, 8000, 0.05 | |
| b1a, b2a, eps = 0.9, 0.999, 1e-8 | |
| t0 = time.time() | |
| for step in range(1, STEPS + 1): | |
| pos = rng.integers(N_CTX, T, size=BATCH) | |
| i1, i2, tgt = data[pos - 2], data[pos - 1], data[pos] | |
| # ---- 前向 ---- | |
| x = np.hstack([emb[i1], emb[i2]]) # (B, IN) | |
| pre = x @ w1.T + b1 # (B, H) | |
| h = np.tanh(pre) | |
| logits = h @ w2.T + b2 # (B, V) | |
| logits -= logits.max(1, keepdims=True) | |
| p = np.exp(logits) | |
| p /= p.sum(1, keepdims=True) | |
| # ---- 损失 ---- | |
| loss = -np.log(p[np.arange(BATCH), tgt] + 1e-12).mean() | |
| # ---- 反向 ---- | |
| dlogits = p.copy() | |
| dlogits[np.arange(BATCH), tgt] -= 1.0 | |
| dlogits /= BATCH | |
| gw2 = dlogits.T @ h | |
| gb2 = dlogits.sum(0) | |
| dh = dlogits @ w2 | |
| dpre = dh * (1.0 - h * h) | |
| gw1 = dpre.T @ x | |
| gb1 = dpre.sum(0) | |
| dx = dpre @ w1 | |
| demb = np.zeros_like(emb) | |
| np.add.at(demb, i1, dx[:, :D]) # 同一个 token 在一批里出现多次, 梯度要累加 | |
| np.add.at(demb, i2, dx[:, D:]) | |
| # ---- Adam(原地更新,不加 L2)---- | |
| lr = LR * (0.5 * (1 + math.cos(math.pi * step / STEPS))) | |
| bc1, bc2 = 1 - b1a ** step, 1 - b2a ** step | |
| for key, P, G in (("e", emb, demb), ("w1", w1, gw1), ("b1", b1, gb1), | |
| ("w2", w2, gw2), ("b2", b2, gb2)): | |
| M[key] = b1a * M[key] + (1 - b1a) * G | |
| S[key] = b2a * S[key] + (1 - b2a) * G * G | |
| P -= lr * (M[key] / bc1) / (np.sqrt(S[key] / bc2) + eps) | |
| if step % 500 == 0 or step == 1: | |
| print(f"step {step:5d} loss {loss:.4f} lr {lr:.4f} ({time.time()-t0:.0f}s)") | |
| json.dump({"name": MODEL_NAME, "params": int(N_PARAMS), "vocab": "".join(chars), | |
| "config": {"n_ctx": N_CTX, "d": D, "h": H}, | |
| "weights": {"emb": emb.tolist(), "w1": w1.tolist(), "b1": b1.tolist(), | |
| "w2": w2.tolist(), "b2": b2.tolist()}}, | |
| open(CKPT, "w", encoding="utf-8"), ensure_ascii=False) | |
| print("已保存:", CKPT) | |
| # ---------------- 采样 ---------------- | |
| def generate(prompt="t", n=160, temperature=0.7, seed=None): | |
| r = random.Random(seed) | |
| out = prompt | |
| ids = [c2i[c] for c in (" " + prompt)[-2:]] | |
| for _ in range(n): | |
| x = np.concatenate([emb[ids[0]], emb[ids[1]]]) | |
| h = np.tanh(w1 @ x + b1) | |
| z = w2 @ h + b2 | |
| z = z / temperature | |
| z -= z.max() | |
| q = np.exp(z); q /= q.sum() | |
| acc, nxt = 0.0, V - 1 | |
| u = r.random() | |
| for i, qi in enumerate(q): | |
| acc += qi | |
| if u <= acc: | |
| nxt = i | |
| break | |
| out += i2c[nxt] | |
| ids = [ids[1], nxt] | |
| return out | |
| print(f"\n--- {MODEL_NAME} 生成样例 ---") | |
| for k, pr in enumerate(["t", "the cat ", "the sun ", "the dog "]): | |
| print(f"[{k+1}] {generate(pr, seed=100 + k)!r}") | |