Text Generation
MLX
English
German
next-concept-modeling
apple-silicon
diffusion-lm
research-code
negative-results
Instructions to use Red-fuchs/ncm-modular with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use Red-fuchs/ncm-modular with MLX:
# Make sure mlx-lm is installed # pip install --upgrade mlx-lm # if on a CUDA device, also pip install mlx[cuda] # Generate text with mlx-lm from mlx_lm import load, generate model, tokenizer = load("Red-fuchs/ncm-modular") prompt = "Once upon a time in" text = generate(model, tokenizer, prompt=prompt, verbose=True) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- MLX LM
How to use Red-fuchs/ncm-modular with MLX LM:
Generate or start a chat session
# Install MLX LM uv tool install mlx-lm # Generate some text mlx_lm.generate --model "Red-fuchs/ncm-modular" --prompt "Once upon a time"
- Atomic Chat
File size: 7,532 Bytes
b5731a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | """Train prefix-conditioned NCM generator on MASSIVE de-DE pairs (frozen wmt14 codec).
usage: train_gen.py --data w/massive_de --out w/ncm_run --steps N [--n 64 (overfit)] [--minutes M]"""
import os, sys, json, time, argparse
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import WORK
import numpy as np, mlx.core as mx, mlx.optimizers as optim
from mlx import nn
from mlx.utils import tree_flatten, tree_unflatten
from transformers import AutoTokenizer
from gen_mlx import *
ap = argparse.ArgumentParser()
ap.add_argument("--data", default=WORK+"/massive_de"); ap.add_argument("--out", default=WORK+"/ncm_run")
ap.add_argument("--steps", type=int, default=20000); ap.add_argument("--minutes", type=float, default=1e9)
ap.add_argument("--bs", type=int, default=64); ap.add_argument("--lr", type=float, default=4e-4)
ap.add_argument("--P", type=int, default=48); ap.add_argument("--D", type=int, default=512)
ap.add_argument("--H", type=int, default=8); ap.add_argument("--nl", type=int, default=8)
ap.add_argument("--n", type=int, default=0, help="overfit on first n examples"); ap.add_argument("--drop", type=float, default=0.1)
ap.add_argument("--corrupt", type=float, default=0.0); ap.add_argument("--eval_every", type=int, default=1000)
ap.add_argument("--ckpt_every", type=int, default=500); ap.add_argument("--resume", action="store_true")
ap.add_argument("--compile", type=int, default=0); ap.add_argument("--fp16", type=int, default=1); ap.add_argument("--scale", type=float, default=1024.)
ap.add_argument("--bench", action="store_true", help="throughput only")
a = ap.parse_args()
EOS, PAD, L = 151645, 151643, 64
os.makedirs(a.out, exist_ok=True)
tok = AutoTokenizer.from_pretrained(WORK+"/tok/Qwen3-0.6B")
fc = FrozenCodec()
def load(sp): return [json.loads(l) for l in open(f"{a.data}/{sp}.jsonl")]
def pad(ids, n): ids = ids[:n]; return ids + [PAD] * (n - len(ids))
def prefix_arr(D): return np.array([pad(tok.encode(d["prefix"])[:a.P - 1] + [EOS], a.P) for d in D], np.int32)
def target_arr(D): return np.array([pad(tok.encode(d["target"])[:L - 1] + [EOS], L) for d in D], np.int32)
def codes_of(T, bs=128):
out = []
for i in range(0, len(T), bs):
cs = fc.encode(mx.array(T[i:i + bs])); out.append(np.array(mx.concatenate(cs, 1))); mx.clear_cache()
return np.concatenate(out)
def get_split(sp, n=0):
cache = f"{a.out}/../cache_{sp}_P{a.P}.npz" if not n else None
D = load(sp)[: n or None]
if cache and os.path.exists(cache): z = np.load(cache); return D, z["p"], z["t"], z["c"]
p, t = prefix_arr(D), target_arr(D); c = codes_of(t)
if cache: np.savez(cache, p=p, t=t, c=c)
return D, p, t, c
def split_levels(flat):
o, out = 0, []
for l in fc.levels: out.append(flat[:, o:o + l]); o += l
return out
def loss_fn(gen, flat, ptoks, corrupt):
codes = split_levels(flat)
seen = codes
if corrupt > 0:
seen = []
for k, c in enumerate(codes):
if k < fc.K - 1:
flip = mx.random.uniform(shape=c.shape) < corrupt
c = mx.where(flip, mx.random.randint(0, 4096, c.shape), c)
seen.append(c)
inputs = [mx.stop_gradient(x) for x in fc.teacher_inputs(seen)]
logits = gen(inputs, ptoks)
ce = [nn.losses.cross_entropy(lg, c, reduction="sum") for lg, c in zip(logits, codes)]
return sum(ce) / (flat.shape[0] * flat.shape[1])
def main():
mx.random.seed(0); np.random.seed(0)
Dtr, ptr, ttr, ctr = get_split("train", a.n)
gen = Gen(fc, a.P, a.D, a.H, a.nl, PAD, drop=a.drop)
print("params", count_params(gen) / 1e6, "M; train", len(Dtr), flush=True)
steps0, ck = 0, f"{a.out}/ckpt.safetensors"
if a.resume and os.path.exists(ck):
gen.load_weights(ck); steps0 = json.load(open(f"{a.out}/ckpt.json"))["step"]; print("resumed", steps0)
sched = optim.join_schedules([optim.linear_schedule(0, a.lr, 200), optim.cosine_decay(a.lr, max(a.steps - 200, 1), a.lr * 0.05)], [200])
opt = optim.AdamW(sched, weight_decay=0.05, betas=(0.9, 0.98)); opt.state["step"] = mx.array(steps0)
# master fp32 weights `gen` (optimizer); compute copy `g16` (fp16) re-cast from master each step; static-ish loss scale
g16 = Gen(fc, a.P, a.D, a.H, a.nl, PAD, drop=a.drop).set_compute(mx.float16 if a.fp16 else mx.float32)
SC = a.scale if a.fp16 else 1.0
vg = nn.value_and_grad(g16, lambda g, f, p: loss_fn(g, f, p, a.corrupt) * SC)
cast = (lambda t: t.astype(mx.float16)) if a.fp16 else (lambda t: t)
from mlx.utils import tree_map
skipped = [0]
def step(f, p):
g16.update(tree_map(cast, gen.parameters()))
loss, grads = vg(g16, f, p)
grads = tree_map(lambda t: t.astype(mx.float32) / SC, grads)
grads, gn = optim.clip_grad_norm(grads, 1.0)
ok = mx.isfinite(gn)
grads = tree_map(lambda t: mx.where(ok, t, 0.0), grads)
opt.update(gen, grads); return loss / SC, ok
state = [gen.state, opt.state, mx.random.state, g16.state]
Ddev = load('dev')[:300]; pdev, tdev = prefix_arr(Ddev), target_arr(Ddev); cdev = codes_of(tdev); best = [-1.0]; curve = []
def dev_loss():
g16.eval(); l = loss_fn(g16, mx.array(cdev), mx.array(pdev), 0.0).item(); g16.train(); return l
N = len(Dtr); t0 = time.time(); ema = None; toks = 0
gen.train()
for s in range(steps0, a.steps):
idx = np.random.randint(0, N, a.bs)
loss, ok = step(mx.array(ctr[idx]), mx.array(ptr[idx])); mx.eval(state, loss, ok); skipped[0] += (not ok.item())
toks += a.bs * (L + a.P)
l = loss.item(); ema = l if ema is None else .98 * ema + .02 * l
if (s + 1) % 50 == 0:
el = time.time() - t0
print(f"step {s+1} loss {ema:.4f} {toks/el:.0f} tok/s(raw P+L) {a.bs*(s+1-steps0)/el:.1f} ex/s skipped {skipped[0]} elapsed {el/60:.1f}m", flush=True)
if a.bench and time.time() - t0 > a.minutes * 60: break
if (s + 1) % a.ckpt_every == 0 or (s + 1) == a.steps or time.time() - t0 > a.minutes * 60:
gen.save_weights(ck); json.dump({"step": s + 1, "loss": ema, "args": vars(a)}, open(f"{a.out}/ckpt.json", "w"))
if (s + 1) % a.eval_every == 0 or (s + 1) == a.steps:
g16.update(tree_map(cast, gen.parameters())); g16.eval(); acc = quick_eval(g16, Dtr, ptr, ttr); accd = quick_eval(g16, Ddev, pdev, tdev, 300); g16.train(); dl = dev_loss()
r = dict(step=s + 1, train_loss_ema=ema, train_exact256=acc, dev_exact300=accd, dev_loss=dl); curve.append(r); print(" ", r, flush=True)
json.dump(curve, open(f"{a.out}/dev_curve.json", "w"))
if accd - 1e-3 * dl > best[0]: best[0] = accd - 1e-3 * dl; gen.save_weights(f"{a.out}/best.safetensors"); json.dump(r, open(f"{a.out}/best.json", "w"))
if time.time() - t0 > a.minutes * 60: print("time budget reached"); break
gen.save_weights(ck); json.dump({"step": s + 1, "loss": ema, "args": vars(a)}, open(f"{a.out}/ckpt.json", "w"))
def quick_eval(gen, D, p, t, n=256):
n = min(n, len(D)); ex = 0
for i in range(0, n, 128):
toks, _ = generate(gen, fc, mx.array(p[i:i + 128][: n - i])); mx.eval(toks)
for r, ref in zip(toks.tolist(), t[i:i + 128]):
r = r[:r.index(EOS)] if EOS in r else r; ref = list(ref); ref = ref[:ref.index(EOS)]
ex += [x for x in r if x != PAD] == ref
return ex / n
def partial_compile(state):
import functools
return lambda f: mx.compile(f, inputs=state, outputs=state)
if __name__ == "__main__": main()
|