GGUFGuy commited on
Commit
2a49d9a
·
0 Parent(s):

Duplicate from GGUFGuy/hyperdex-trainer

Browse files
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ runtime/
2
+ __pycache__/
3
+ *.pyc
4
+ .venv/
Dockerfile ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.12-slim
2
+
3
+ ENV PYTHONUNBUFFERED=1 \
4
+ PYTHONDONTWRITEBYTECODE=1 \
5
+ PIP_NO_CACHE_DIR=1 \
6
+ HF_HOME=/home/user/.cache/huggingface \
7
+ HF_HUB_ENABLE_HF_TRANSFER=0 \
8
+ PORT=7860
9
+
10
+ RUN apt-get update && apt-get install -y --no-install-recommends \
11
+ build-essential git curl && \
12
+ rm -rf /var/lib/apt/lists/*
13
+
14
+ RUN useradd -m -u 1000 user
15
+ WORKDIR /home/user/app
16
+
17
+ COPY --chown=user requirements.txt .
18
+ RUN pip install --upgrade pip && pip install -r requirements.txt
19
+
20
+ COPY --chown=user . .
21
+
22
+ RUN mkdir -p /home/user/app/runtime /home/user/.cache/huggingface && \
23
+ chown -R user:user /home/user
24
+
25
+ USER user
26
+ EXPOSE 7860
27
+
28
+ CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860", \
29
+ "--workers", "1", "--timeout-keep-alive", "75"]
README.md ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: NanoDex
3
+ emoji: 🧬
4
+ colorFrom: green
5
+ colorTo: blue
6
+ sdk: docker
7
+ app_port: 7860
8
+ pinned: true
9
+ license: apache-2.0
10
+ short_description: Train a decoder-only LM from scratch on fineweb-edu
11
+ hf_oauth: true
12
+ hf_oauth_scopes:
13
+ - read-repos
14
+ - write-repos
15
+ - manage-repos
16
+ hf_oauth_expiration_minutes: 480
17
+ startup_duration_timeout: 45m
18
+ datasets:
19
+ - HuggingFaceFW/fineweb-edu
20
+ ---
21
+
22
+ # 🧬 NanoDex
23
+
24
+ **Train your first decoder-only language model from scratch, in the cloud.**
25
+
26
+ Not a fine-tune. Not an adapter. Random weights, a real corpus, and a real loss
27
+ curve you watch fall in real time.
28
+
29
+ Pick a size (**492k / 1.06M / 5.17M / 8.06M** parameters), pick a token budget
30
+ (**200M – 1.5B** tokens of fineweb-edu), and queue it. A worker picks it up, you
31
+ watch the loss fall live, and when it's done the model is pushed to *your*
32
+ Hugging Face account with a generated model card.
33
+
34
+ ## Pages
35
+
36
+ | Route | What it is |
37
+ |---|---|
38
+ | `/` | Landing page when signed out; your dashboard when signed in |
39
+ | `/train` | Four-step run wizard — size, tokens, name, review |
40
+ | `/models` | Everything you've trained: training, queued (with position), ready |
41
+ | `/models/{id}` | One run in detail — live loss chart, metrics, logs, publish, playground |
42
+ | `/queue` | Global queue and worker status |
43
+ | `/profile` | Account, totals, permissions |
44
+ | `/about` | The full recipe |
45
+
46
+ ## Architecture
47
+
48
+ A standard modern decoder-only transformer (`LlamaForCausalLM`): SiLU MLP,
49
+ RMSNorm (ε=1e-5), rotary position embeddings (θ=10000), grouped-query attention,
50
+ tied input/output embeddings, no biases. The four tiers are that same recipe
51
+ scaled down in width and depth.
52
+
53
+ | Tier | Parameters | Layers | Hidden | Heads (KV) | FFN | Tokens / step |
54
+ |---|---:|---:|---:|---|---:|---:|
55
+ | NanoDex-500k | 492,192 | 3 | 96 | 6 (2) | 256 | 131,072 |
56
+ | NanoDex-1M | 1,062,272 | 5 | 128 | 8 (4) | 288 | 262,144 |
57
+ | NanoDex-5M | 5,172,384 | 9 | 224 | 8 (2) | 592 | 393,216 |
58
+ | NanoDex-8M | 8,060,256 | 9 | 288 | 9 (3) | 704 | 524,288 |
59
+
60
+ A typical small-LM vocabulary (~49k tokens) would be 28M embedding parameters on
61
+ its own — more than three times the biggest model here. So NanoDex ships its own
62
+ **2,048-token byte-level BPE** trained on fineweb-edu (~2.7 chars/token). The
63
+ parameter counts above are real totals, embeddings included.
64
+
65
+ ## Data
66
+
67
+ [`HuggingFaceFW/fineweb-edu`](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu),
68
+ config `sample-10BT`. A 1.505B-token slice is tokenized **once** into a flat
69
+ `uint16` file (~2.8 GB) that every run reads from, so workers are never
70
+ data-bound. Each run starts at its own random offset. Building the cache is a
71
+ one-time cost per container; a run asking for more tokens than are cached yet
72
+ waits for the writer and reports it in the run log.
73
+
74
+ ## Training
75
+
76
+ AdamW (β=0.9/0.95, wd=0.1, grad-clip 1.0) · 2% warmup then cosine decay to 10%
77
+ of peak · bf16 autocast · 512-token context · 128–256 sequences per forward
78
+ pass · 131k–524k tokens per optimizer step, with automatic micro-batch backoff
79
+ if CUDA runs out of memory. One worker thread per GPU claims the oldest queued run
80
+ atomically, so a 4-GPU Space trains four models at once.
81
+
82
+ ## Layout
83
+
84
+ ```
85
+ main.py FastAPI app — pages, JSON API, Hugging Face OAuth
86
+ Dockerfile
87
+ web/templates/ landing, home, train wizard, models, detail, queue,
88
+ profile, about, error
89
+ web/static/ app.css, app.js (dependency-free charts + polling)
90
+ nanodex/config.py tier definitions + exact parameter counting
91
+ nanodex/data.py shared fineweb-edu token cache
92
+ nanodex/trainer.py the training loop
93
+ nanodex/worker.py one worker per GPU
94
+ nanodex/hub.py push a finished run to the user's account
95
+ nanodex/estimate.py ETA, self-calibrating from completed runs
96
+ nanodex/db.py SQLite job store
97
+ tokenizer/ the 2,048-token BPE
98
+ ```
99
+
100
+ ## A note on scale
101
+
102
+ 1.5 billion tokens into an 8M-parameter model is ~9× past Chinchilla-optimal,
103
+ and into a 500k-parameter model over 150× past it — on purpose. Tiny models keep
104
+ improving long past the compute-optimal point, and the goal here isn't FLOP
105
+ efficiency — it's watching cross-entropy fall from 7.6
106
+ (uniform over 2,048 tokens) toward 4, and seeing a network that started as pure
107
+ noise begin to emit English words.
108
+
109
+ Nothing trained here is a useful assistant. That was never the point.
110
+
111
+ Built with 🤗 by [hugging-science](https://huggingface.co/hugging-science).
main.py ADDED
@@ -0,0 +1,643 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NanoDex — a place to pre-train small decoder-only language models from scratch.
3
+
4
+ FastAPI multi-page app:
5
+ / landing (logged out) / dashboard (logged in)
6
+ /train multi-step run wizard
7
+ /models your runs: training, queued, finished
8
+ /models/{id} one run in detail — live loss, logs, publish, playground
9
+ /profile account + stats
10
+ /login /logout Hugging Face OAuth
11
+ /api/* JSON for live polling
12
+ """
13
+
14
+ import os
15
+ import secrets
16
+ import sys
17
+ import threading
18
+ import time
19
+ from typing import Optional
20
+
21
+ # Printed before anything heavy so a stalled boot is always diagnosable from
22
+ # the Space's run log.
23
+ print("[boot] NanoDex starting", flush=True)
24
+
25
+ from fastapi import FastAPI, Request, HTTPException, Body
26
+ from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
27
+ from fastapi.staticfiles import StaticFiles
28
+ from fastapi.templating import Jinja2Templates
29
+ from starlette.middleware.sessions import SessionMiddleware
30
+
31
+ from nanodex import db, worker, hub, estimate, store
32
+ from nanodex import data as ndata
33
+ from nanodex.config import (TIERS, TIER_ORDER, MIN_TOKENS, MAX_TOKENS,
34
+ TOKEN_STOPS_M, SEQ_LEN, VOCAB_SIZE)
35
+ from nanodex.trainer import total_steps_for
36
+
37
+ BASE = os.path.dirname(os.path.abspath(__file__))
38
+ TOKENIZER_DIR = os.path.join(BASE, "tokenizer")
39
+ MAX_ACTIVE_PER_USER = int(os.environ.get("MAX_ACTIVE_PER_USER", "2"))
40
+
41
+ OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID")
42
+ OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET")
43
+ OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES", "openid profile read-repos write-repos manage-repos")
44
+ OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co")
45
+ SPACE_HOST = os.environ.get("SPACE_HOST")
46
+ OAUTH_READY = bool(OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET)
47
+
48
+ print(f"[boot] data dir: {db.DATA_DIR}", flush=True)
49
+ db.init()
50
+ print("[boot] database ready", flush=True)
51
+
52
+ from tokenizers import Tokenizer as _FastTok
53
+ TOKENIZER = _FastTok.from_file(os.path.join(TOKENIZER_DIR, "tokenizer.json"))
54
+ print(f"[boot] tokenizer loaded ({TOKENIZER.get_vocab_size()} tokens)", flush=True)
55
+
56
+ DEVICES = worker.devices()
57
+ print(f"[boot] devices: {DEVICES}", flush=True)
58
+
59
+
60
+ def _restore_history():
61
+ """
62
+ Pull previously finished runs back out of the bucket. Done in a thread so
63
+ a slow or unreachable bucket never delays the app coming up.
64
+ """
65
+ try:
66
+ if not store.available():
67
+ return
68
+ known = db.known_job_ids()
69
+ ids = [i for i in store.list_runs() if i not in known]
70
+ if not ids:
71
+ print("[store] no archived runs to restore", flush=True)
72
+ return
73
+ n = 0
74
+ for jid in ids:
75
+ meta = store.fetch_run_meta(jid)
76
+ if meta and db.import_run(meta):
77
+ n += 1
78
+ store._state["restored"] = n
79
+ print(f"[store] restored {n} run(s) from durable storage", flush=True)
80
+ except Exception as exc:
81
+ print(f"[store] history restore failed: {exc!r}", flush=True)
82
+
83
+
84
+ threading.Thread(target=_restore_history, name="restore-history",
85
+ daemon=True).start()
86
+ ndata.start_background_build(TOKENIZER)
87
+ worker.start_workers(TOKENIZER_DIR)
88
+ ON_GPU = DEVICES[0].startswith("cuda")
89
+ INFER_DEVICE = DEVICES[0] if ON_GPU else "cpu"
90
+
91
+ app = FastAPI(title="NanoDex")
92
+ IN_SPACE = bool(SPACE_HOST)
93
+ app.add_middleware(
94
+ SessionMiddleware,
95
+ secret_key=os.environ.get("SESSION_SECRET") or (OAUTH_CLIENT_SECRET or secrets.token_hex(32)),
96
+ # A Space is served inside an iframe on huggingface.co, which makes the
97
+ # session cookie third-party: "lax" gets dropped on the redirect back from
98
+ # the OAuth provider and the CSRF state check fails. "none" + Secure is the
99
+ # only combination that survives that round trip.
100
+ same_site="none" if IN_SPACE else "lax",
101
+ https_only=IN_SPACE,
102
+ max_age=8 * 3600,
103
+ )
104
+ app.mount("/static", StaticFiles(directory=os.path.join(BASE, "web/static")), name="static")
105
+ templates = Jinja2Templates(directory=os.path.join(BASE, "web/templates"))
106
+
107
+ # --------------------------------------------------------------- oauth ------
108
+ oauth = None
109
+ if OAUTH_READY:
110
+ from authlib.integrations.starlette_client import OAuth
111
+ oauth = OAuth()
112
+ oauth.register(
113
+ name="huggingface",
114
+ client_id=OAUTH_CLIENT_ID,
115
+ client_secret=OAUTH_CLIENT_SECRET,
116
+ server_metadata_url=f"{OPENID_PROVIDER_URL}/.well-known/openid-configuration",
117
+ client_kwargs={"scope": OAUTH_SCOPES},
118
+ )
119
+
120
+
121
+ def current_user(request: Request) -> Optional[dict]:
122
+ u = request.session.get("user")
123
+ if u and u.get("expires_at", 0) > time.time():
124
+ return u
125
+ if u:
126
+ request.session.pop("user", None)
127
+ return None
128
+
129
+
130
+ def require_user(request: Request) -> dict:
131
+ u = current_user(request)
132
+ if not u:
133
+ raise HTTPException(401, "not signed in")
134
+ return u
135
+
136
+
137
+ def _redirect_uri(request: Request) -> str:
138
+ if SPACE_HOST:
139
+ return f"https://{SPACE_HOST}/auth/callback"
140
+ return str(request.url_for("auth_callback"))
141
+
142
+
143
+ @app.get("/login")
144
+ async def login(request: Request, next: str = "/"):
145
+ request.session["next"] = next
146
+ if not OAUTH_READY:
147
+ # Local development only — never reachable on a Space.
148
+ request.session["user"] = {
149
+ "username": os.environ.get("DEV_USER", "localdev"),
150
+ "name": "Local Dev", "picture": None, "token": None,
151
+ "expires_at": time.time() + 8 * 3600,
152
+ }
153
+ return RedirectResponse(next, status_code=303)
154
+ return await oauth.huggingface.authorize_redirect(request, _redirect_uri(request))
155
+
156
+
157
+ @app.get("/auth/callback", name="auth_callback")
158
+ async def auth_callback(request: Request):
159
+ nxt = request.session.get("next", "/") or "/"
160
+ try:
161
+ token = await oauth.huggingface.authorize_access_token(request)
162
+ except Exception as exc:
163
+ if "state" in str(exc).lower() and request.cookies.get("nx_retry") != "1":
164
+ # The state cookie didn't survive the round trip. Try once more
165
+ # with a fresh session; the marker cookie stops this from looping.
166
+ request.session.clear()
167
+ r = RedirectResponse(f"/login?next={nxt}", status_code=303)
168
+ r.set_cookie("nx_retry", "1", max_age=120, path="/",
169
+ httponly=True, secure=IN_SPACE,
170
+ samesite="none" if IN_SPACE else "lax")
171
+ return r
172
+ return render(request, "error.html", status_code=400,
173
+ code="Sign-in failed", detail=str(exc)[:300])
174
+ info = token.get("userinfo") or await oauth.huggingface.userinfo(token=token)
175
+ request.session["user"] = {
176
+ "username": info.get("preferred_username") or info.get("name"),
177
+ "name": info.get("name") or info.get("preferred_username"),
178
+ "picture": info.get("picture"),
179
+ "token": token.get("access_token"),
180
+ "expires_at": time.time() + 8 * 3600,
181
+ }
182
+ r = RedirectResponse(nxt, status_code=303)
183
+ r.delete_cookie("nx_retry", path="/")
184
+ return r
185
+
186
+
187
+ @app.get("/logout")
188
+ async def logout(request: Request):
189
+ request.session.clear()
190
+ return RedirectResponse("/", status_code=303)
191
+
192
+
193
+ # ---------------------------------------------------------- serialization ---
194
+ def job_dto(j, with_position=False):
195
+ # A row written before a tier list change would otherwise KeyError here and
196
+ # take down the whole page, so fall back to the first tier for display.
197
+ tier = TIERS.get(j["tier"]) or TIERS[TIER_ORDER[0]]
198
+ target = max(int(j["target_tokens"]), 1)
199
+ # Step counts rarely divide the budget exactly, so a finished run would
200
+ # otherwise sit at 99.9% forever.
201
+ progress = (1.0 if j["status"] == "done"
202
+ else min(1.0, (j["tokens_seen"] or 0) / target))
203
+ d = {
204
+ "id": j["id"],
205
+ "name": j["model_name"],
206
+ "username": j["username"],
207
+ "tier": j["tier"],
208
+ "tier_label": tier.label,
209
+ "n_params": j["n_params"],
210
+ "status": j["status"],
211
+ "target_tokens": j["target_tokens"],
212
+ "tokens_seen": j["tokens_seen"] or 0,
213
+ "progress": progress,
214
+ "step": j["step"] or 0,
215
+ "total_steps": j["total_steps"] or 0,
216
+ "loss": j["loss"],
217
+ "best_loss": j["best_loss"],
218
+ "lr": j["lr"],
219
+ "tok_per_s": j["tok_per_s"] or 0,
220
+ "eta_s": j["eta_s"],
221
+ "gpu_index": j["gpu_index"],
222
+ "error": j["error"],
223
+ "pushed_repo": j["pushed_repo"],
224
+ "live_step": j["live_step"],
225
+ "live_at": j["live_at"],
226
+ "has_checkpoint": bool(j["live_step"] is not None) or j["status"] == "done",
227
+ "created_at": j["created_at"],
228
+ "started_at": j["started_at"],
229
+ "finished_at": j["finished_at"],
230
+ "arch": {
231
+ "layers": tier.num_hidden_layers, "hidden": tier.hidden_size,
232
+ "heads": tier.num_attention_heads, "kv_heads": tier.num_key_value_heads,
233
+ "ffn": tier.intermediate_size, "head_dim": tier.head_dim,
234
+ "ctx": SEQ_LEN, "vocab": VOCAB_SIZE,
235
+ },
236
+ }
237
+ if with_position and j["status"] == "queued":
238
+ d["queue_position"] = db.queue_position(j["id"])
239
+ return d
240
+
241
+
242
+ def _tok_label(n):
243
+ return f"{n / 1e9:g}B" if n >= 1_000_000_000 else f"{n / 1e6:g}M"
244
+
245
+
246
+ def tiers_dto():
247
+ out = []
248
+ for k in TIER_ORDER:
249
+ t = TIERS[k]
250
+ out.append({
251
+ "key": k, "label": t.label, "params": t.param_count(),
252
+ "layers": t.num_hidden_layers, "hidden": t.hidden_size,
253
+ "heads": t.num_attention_heads, "kv_heads": t.num_key_value_heads,
254
+ "ffn": t.intermediate_size, "batch_tokens": t.batch_tokens, "lr": t.lr,
255
+ })
256
+ return out
257
+
258
+
259
+ # ------------------------------------------------------------- pages --------
260
+ def render(request: Request, template: str, status_code: int = 200, **kw):
261
+ base = {"user": current_user(request),
262
+ "on_gpu": ON_GPU, "n_devices": len(DEVICES)}
263
+ base.update(kw)
264
+ return templates.TemplateResponse(request, template, base,
265
+ status_code=status_code)
266
+
267
+
268
+ @app.get("/", response_class=HTMLResponse)
269
+ async def index(request: Request):
270
+ u = current_user(request)
271
+ if not u:
272
+ return render(request, "landing.html", stats=db.stats(),
273
+ tiers=tiers_dto(), min_tokens=MIN_TOKENS,
274
+ max_tokens=MAX_TOKENS,
275
+ recent=[job_dto(j) for j in db.list_public(limit=6)])
276
+ jobs = [job_dto(j, True) for j in db.list_jobs(username=u["username"], limit=8)]
277
+ return render(request, "home.html", jobs=jobs, stats=db.stats())
278
+
279
+
280
+ @app.get("/train", response_class=HTMLResponse)
281
+ async def train_page(request: Request):
282
+ if not current_user(request):
283
+ return RedirectResponse("/login?next=/train", status_code=303)
284
+ return render(request, "train.html", tiers=tiers_dto(),
285
+ min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS,
286
+ stops=TOKEN_STOPS_M,
287
+ max_active=MAX_ACTIVE_PER_USER, seq_len=SEQ_LEN,
288
+ vocab=VOCAB_SIZE)
289
+
290
+
291
+ @app.get("/models", response_class=HTMLResponse)
292
+ async def models_page(request: Request):
293
+ if not current_user(request):
294
+ return RedirectResponse("/login?next=/models", status_code=303)
295
+ return render(request, "models.html")
296
+
297
+
298
+ @app.get("/models/{job_id}", response_class=HTMLResponse)
299
+ async def model_detail(request: Request, job_id: str):
300
+ # Readable by anyone — browsing the shelf shouldn't need a sign-in.
301
+ # Generating, cancelling and publishing still do.
302
+ j = db.get_job(job_id)
303
+ if not j:
304
+ return render(request, "error.html", status_code=404,
305
+ code="404", detail="No run with that id.")
306
+ me = current_user(request)
307
+ return render(request, "detail.html", job=job_dto(j, True),
308
+ infer_device=INFER_DEVICE,
309
+ owner=bool(me and j["username"] == me["username"]))
310
+
311
+
312
+ @app.get("/explore", response_class=HTMLResponse)
313
+ async def explore_page(request: Request):
314
+ return render(request, "explore.html", tiers=tiers_dto())
315
+
316
+
317
+ @app.get("/u/{username}", response_class=HTMLResponse)
318
+ async def public_profile(request: Request, username: str):
319
+ prof = db.user_profile(username)
320
+ if not prof:
321
+ return render(request, "error.html", status_code=404, code="404",
322
+ detail=f"@{username} hasn't trained anything here yet.")
323
+ me = current_user(request)
324
+ return render(request, "user.html", profile=prof, tiers=tiers_dto(),
325
+ is_me=bool(me and me["username"] == username))
326
+
327
+
328
+ @app.get("/queue", response_class=HTMLResponse)
329
+ async def queue_page(request: Request):
330
+ return render(request, "queue.html")
331
+
332
+
333
+ @app.get("/profile", response_class=HTMLResponse)
334
+ async def profile_page(request: Request):
335
+ u = current_user(request)
336
+ if not u:
337
+ return RedirectResponse("/login?next=/profile", status_code=303)
338
+ jobs = db.list_jobs(username=u["username"], limit=500)
339
+ done = [j for j in jobs if j["status"] == "done"]
340
+ return render(request, "profile.html", summary={
341
+ "total": len(jobs), "done": len(done),
342
+ "active": sum(1 for j in jobs if j["status"] in ("queued", "running")),
343
+ "tokens": sum(j["tokens_seen"] or 0 for j in jobs),
344
+ "params": sum(j["n_params"] for j in done),
345
+ "published": sum(1 for j in done if j["pushed_repo"]),
346
+ "best": min([j["best_loss"] for j in done if j["best_loss"]], default=None),
347
+ },
348
+ jobs=[job_dto(j, True) for j in jobs[:20]])
349
+
350
+
351
+ @app.get("/about", response_class=HTMLResponse)
352
+ async def about_page(request: Request):
353
+ return render(request, "about.html", tiers=tiers_dto(),
354
+ min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS,
355
+ seq_len=SEQ_LEN, vocab=VOCAB_SIZE,
356
+ cache_target=ndata.TARGET_TOKENS,
357
+ max_active=MAX_ACTIVE_PER_USER)
358
+
359
+
360
+ # --------------------------------------------------------------- api --------
361
+ @app.get("/api/me")
362
+ async def api_me(request: Request):
363
+ u = current_user(request)
364
+ if not u:
365
+ return {"signed_in": False}
366
+ return {"signed_in": True, "username": u["username"],
367
+ "name": u["name"], "picture": u["picture"]}
368
+
369
+
370
+ @app.get("/api/tiers")
371
+ async def api_tiers():
372
+ return {"tiers": tiers_dto(), "min_tokens": MIN_TOKENS,
373
+ "max_tokens": MAX_TOKENS, "stops_m": TOKEN_STOPS_M}
374
+
375
+
376
+ @app.get("/api/estimate")
377
+ async def api_estimate(tier: str, tokens: int):
378
+ if tier not in TIERS:
379
+ raise HTTPException(400, "unknown tier")
380
+ tokens = max(MIN_TOKENS, min(MAX_TOKENS, int(tokens)))
381
+ s = estimate.summary(tier, tokens, ON_GPU)
382
+ return {**s, "eta_human": estimate.human_time(s["eta_s"]),
383
+ "chinchilla_x": tokens / (s["n_params"] * 20)}
384
+
385
+
386
+ @app.get("/api/jobs")
387
+ async def api_jobs(request: Request):
388
+ u = require_user(request)
389
+ return {"jobs": [job_dto(j, True)
390
+ for j in db.list_jobs(username=u["username"], limit=200)]}
391
+
392
+
393
+ @app.get("/api/jobs/{job_id}")
394
+ async def api_job(job_id: str):
395
+ j = db.get_job(job_id)
396
+ if not j:
397
+ raise HTTPException(404, "no such run")
398
+ return job_dto(j, True)
399
+
400
+
401
+ @app.get("/api/jobs/{job_id}/metrics")
402
+ async def api_metrics(job_id: str):
403
+ return {"metrics": db.get_metrics(job_id)}
404
+
405
+
406
+ @app.get("/api/jobs/{job_id}/logs")
407
+ async def api_logs(job_id: str):
408
+ return {"logs": db.get_logs(job_id, 300)}
409
+
410
+
411
+ @app.post("/api/jobs")
412
+ async def api_create(request: Request, payload: dict = Body(...)):
413
+ u = require_user(request)
414
+ tier = payload.get("tier")
415
+ if tier not in TIERS:
416
+ raise HTTPException(400, "Pick one of the four model sizes.")
417
+ try:
418
+ tokens = int(payload.get("tokens") or 0)
419
+ except (TypeError, ValueError):
420
+ raise HTTPException(400, "Token budget must be a number.")
421
+ if not (MIN_TOKENS <= tokens <= MAX_TOKENS):
422
+ raise HTTPException(400, f"Token budget must be between "
423
+ f"{_tok_label(MIN_TOKENS)} and {_tok_label(MAX_TOKENS)}.")
424
+ if db.user_active_count(u["username"]) >= MAX_ACTIVE_PER_USER:
425
+ raise HTTPException(429, f"You already have {MAX_ACTIVE_PER_USER} runs in "
426
+ "flight. Wait for one to finish, or cancel it.")
427
+
428
+ import re
429
+ name = re.sub(r"[^A-Za-z0-9._-]+", "-", (payload.get("name") or "").strip()).strip("-._")[:80]
430
+ if not name:
431
+ name = f"{TIERS[tier].label.lower()}-{tokens // 10**6}m"
432
+
433
+ job_id = db.create_job(
434
+ username=u["username"], display_name=u["name"], avatar_url=u["picture"],
435
+ model_name=name, tier=tier, n_params=TIERS[tier].param_count(),
436
+ target_tokens=tokens, total_steps=total_steps_for(tier, tokens),
437
+ seed=int(time.time()) % 100000,
438
+ )
439
+ db.add_log(job_id, f"queued by @{u['username']}")
440
+ return {"id": job_id, "position": db.queue_position(job_id)}
441
+
442
+
443
+ @app.post("/api/jobs/{job_id}/cancel")
444
+ async def api_cancel(request: Request, job_id: str):
445
+ u = require_user(request)
446
+ j = db.get_job(job_id)
447
+ if not j:
448
+ raise HTTPException(404, "no such run")
449
+ if j["username"] != u["username"]:
450
+ raise HTTPException(403, "not your run")
451
+ worker.request_cancel(job_id)
452
+ return {"ok": True}
453
+
454
+
455
+ @app.post("/api/jobs/{job_id}/publish")
456
+ def api_publish(request: Request, job_id: str, payload: dict = Body(default={})):
457
+ u = require_user(request)
458
+ j = db.get_job(job_id)
459
+ if not j:
460
+ raise HTTPException(404, "no such run")
461
+ if j["username"] != u["username"]:
462
+ raise HTTPException(403, "not your run")
463
+ if j["status"] != "done":
464
+ raise HTTPException(400, "This run hasn't finished yet.")
465
+ if not u.get("token"):
466
+ raise HTTPException(401, "Sign in again — your token expired.")
467
+ out_dir = os.path.join(db.RUNS_DIR, job_id)
468
+ if not os.path.exists(os.path.join(out_dir, "model.safetensors")):
469
+ store.fetch_checkpoint(job_id, out_dir)
470
+ if not os.path.exists(os.path.join(out_dir, "model.safetensors")):
471
+ raise HTTPException(410, "Checkpoint could not be restored from storage.")
472
+ import re
473
+ name = re.sub(r"[^A-Za-z0-9._-]+", "-",
474
+ (payload.get("repo") or j["model_name"]).strip()).strip("-._")[:80]
475
+ try:
476
+ repo_id = hub.push_run(out_dir, j, u["token"], name or j["model_name"])
477
+ except Exception as exc:
478
+ raise HTTPException(500, f"Upload failed: {exc}")
479
+ db.update_job(job_id, pushed_repo=repo_id)
480
+ return {"repo": repo_id, "url": f"https://huggingface.co/{repo_id}"}
481
+
482
+
483
+ # One cached inference model, keyed by (job, checkpoint) so a newer snapshot
484
+ # replaces it automatically. Generation runs on the GPU when there is one —
485
+ # a 3M-parameter model costs nothing next to the training job sharing it.
486
+ _pg = {"key": None, "model": None}
487
+ _pg_lock = threading.Lock()
488
+
489
+
490
+ def _resolve_checkpoint(j):
491
+ """Final weights if the run finished, otherwise the newest live snapshot."""
492
+ final = os.path.join(db.RUNS_DIR, j["id"])
493
+ if j["status"] == "done":
494
+ if not os.path.exists(os.path.join(final, "model.safetensors")):
495
+ # The container was rebuilt since this run finished — pull the
496
+ # weights back out of the bucket.
497
+ store.fetch_checkpoint(j["id"], final)
498
+ if os.path.exists(os.path.join(final, "model.safetensors")):
499
+ return final, ("final", j["step"])
500
+ live = os.path.join(final, "live")
501
+ if os.path.exists(os.path.join(live, "model.safetensors")):
502
+ return live, ("live", j["live_step"])
503
+ return None, (None, None)
504
+
505
+
506
+ @app.post("/api/jobs/{job_id}/generate")
507
+ def api_generate(request: Request, job_id: str, payload: dict = Body(...)):
508
+ # Anyone signed in can generate from anyone's model — that is the point of
509
+ # a shared shelf. Cancelling and publishing stay with the owner.
510
+ require_user(request)
511
+ j = db.get_job(job_id)
512
+ if not j:
513
+ raise HTTPException(404, "no such run")
514
+
515
+ path, (kind, step) = _resolve_checkpoint(j)
516
+ if not path:
517
+ if j["status"] == "queued":
518
+ raise HTTPException(409, "This run hasn't started yet — "
519
+ "the first checkpoint appears a few steps in.")
520
+ if j["status"] == "running":
521
+ raise HTTPException(409, "No checkpoint saved yet. The first one lands "
522
+ "within the first few percent of the run.")
523
+ raise HTTPException(410, "Checkpoint is gone — the Space restarted.")
524
+
525
+ import torch
526
+ from transformers import AutoModelForCausalLM
527
+
528
+ key = (path, step)
529
+ with _pg_lock:
530
+ if _pg["key"] != key:
531
+ m = AutoModelForCausalLM.from_pretrained(path, dtype=torch.float32)
532
+ m.config.use_cache = True
533
+ _pg["model"] = m.eval().to(INFER_DEVICE)
534
+ _pg["key"] = key
535
+ model = _pg["model"]
536
+
537
+ prompt = payload.get("prompt") or " "
538
+ ids = TOKENIZER.encode(prompt).ids[-(SEQ_LEN - 1):] or [0]
539
+ temp = float(payload.get("temperature", 0.85))
540
+ x = torch.tensor([ids], dtype=torch.long, device=INFER_DEVICE)
541
+ t0 = time.time()
542
+ with torch.no_grad():
543
+ out = model.generate(
544
+ x,
545
+ max_new_tokens=max(1, min(int(payload.get("max_new_tokens", 80)), 256)),
546
+ do_sample=temp > 0.01, temperature=max(temp, 1e-4),
547
+ top_k=int(payload.get("top_k", 40)),
548
+ top_p=float(payload.get("top_p", 0.95)),
549
+ pad_token_id=1, eos_token_id=None,
550
+ )
551
+ ids_out = out[0].tolist()
552
+
553
+ return {
554
+ "text": TOKENIZER.decode(ids_out),
555
+ "prompt": prompt,
556
+ "continuation": TOKENIZER.decode(ids_out[len(ids):]),
557
+ "checkpoint": kind,
558
+ "step": step,
559
+ "total_steps": j["total_steps"],
560
+ "device": INFER_DEVICE,
561
+ "ms": round((time.time() - t0) * 1000),
562
+ "new_tokens": len(ids_out) - len(ids),
563
+ }
564
+
565
+
566
+ @app.get("/api/explore")
567
+ async def api_explore(tier: str = "", sort: str = "recent",
568
+ limit: int = 60, offset: int = 0, username: str = ""):
569
+ limit = max(1, min(limit, 120))
570
+ rows = db.list_public(tier=tier or None, sort=sort, limit=limit,
571
+ offset=max(0, offset), username=username or None)
572
+ return {"models": [job_dto(j) for j in rows],
573
+ "active": [job_dto(j, True) for j in db.list_active_public()],
574
+ "stats": db.stats()}
575
+
576
+
577
+ @app.get("/api/leaderboard")
578
+ async def api_leaderboard():
579
+ lb = db.leaderboard()
580
+ return {"tiers": [
581
+ {"tier": k, "label": (TIERS.get(k) or TIERS[TIER_ORDER[0]]).label,
582
+ "order": TIER_ORDER.index(k) if k in TIER_ORDER else 99,
583
+ "models": [job_dto(j) for j in v]}
584
+ for k, v in sorted(lb.items(),
585
+ key=lambda kv: TIER_ORDER.index(kv[0])
586
+ if kv[0] in TIER_ORDER else 99)]}
587
+
588
+
589
+ @app.get("/api/users/{username}")
590
+ async def api_user(username: str):
591
+ prof = db.user_profile(username)
592
+ if not prof:
593
+ raise HTTPException(404, "no such trainer")
594
+ return {"profile": prof,
595
+ "models": [job_dto(j) for j in
596
+ db.list_public(username=username, limit=100)],
597
+ "active": [job_dto(j, True) for j in
598
+ db.list_jobs(username=username, limit=100)
599
+ if j["status"] in ("queued", "running")]}
600
+
601
+
602
+ @app.get("/api/trainers")
603
+ async def api_trainers():
604
+ return {"trainers": db.top_trainers()}
605
+
606
+
607
+ @app.get("/api/queue")
608
+ async def api_queue():
609
+ infos = worker.worker_info()
610
+ return {
611
+ "workers": [{"index": w["index"], "device": w["device"],
612
+ "job": job_dto(w["job"]) if w["job"] else None}
613
+ for w in infos],
614
+ "running": [job_dto(j) for j in db.list_jobs(status=["running"], limit=64)],
615
+ "queued": [job_dto(j, True) for j in db.list_jobs(status=["queued"], limit=64)],
616
+ "recent": [job_dto(j) for j in db.list_jobs(status=["done"], limit=12)],
617
+ "stats": db.stats(),
618
+ "data": {**ndata.state(), "target": ndata.TARGET_TOKENS},
619
+ "on_gpu": ON_GPU,
620
+ "store": {**store.state(), "bucket": store.BUCKET_ID},
621
+ }
622
+
623
+
624
+ @app.get("/api/health")
625
+ async def health():
626
+ return {"ok": True, "devices": DEVICES, "data": ndata.state(),
627
+ "stats": db.stats(), "store": store.state(),
628
+ "bucket": store.BUCKET_ID}
629
+
630
+
631
+ @app.exception_handler(HTTPException)
632
+ async def http_error(request: Request, exc: HTTPException):
633
+ if request.url.path.startswith("/api/"):
634
+ return JSONResponse({"error": exc.detail}, status_code=exc.status_code)
635
+ if exc.status_code == 401:
636
+ return RedirectResponse(f"/login?next={request.url.path}", status_code=303)
637
+ return render(request, "error.html", status_code=exc.status_code,
638
+ code=str(exc.status_code), detail=exc.detail)
639
+
640
+
641
+ if __name__ == "__main__":
642
+ import uvicorn
643
+ uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))
nanodex/__init__.py ADDED
File without changes
nanodex/config.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Model tier definitions for NanoDex — small decoder-only language models
3
+ trained from scratch.
4
+
5
+ All four tiers share one standard modern transformer recipe, scaled down
6
+ in width and depth:
7
+ - LlamaForCausalLM, SiLU MLP, RMSNorm (eps 1e-5), RoPE theta 10000
8
+ - Grouped-query attention, no attention/MLP biases
9
+ - Tied input/output embeddings
10
+
11
+ A typical small-LM vocabulary (~49k tokens) would be ~28M embedding
12
+ parameters on its own, which makes a 500k-parameter model impossible.
13
+ NanoDex therefore uses a purpose-built 2,048-token BPE trained on
14
+ fineweb-edu, so the advertised size is the REAL total parameter count
15
+ (embeddings included), not "non-embedding params".
16
+ """
17
+
18
+ from dataclasses import dataclass
19
+
20
+ VOCAB_SIZE = 2048
21
+ MAX_POSITION_EMBEDDINGS = 512
22
+ SEQ_LEN = 512
23
+ RMS_NORM_EPS = 1e-5
24
+ ROPE_THETA = 10000.0
25
+
26
+
27
+ @dataclass
28
+ class Tier:
29
+ key: str
30
+ label: str
31
+ hidden_size: int
32
+ num_hidden_layers: int
33
+ num_attention_heads: int
34
+ num_key_value_heads: int
35
+ intermediate_size: int
36
+ # Peak learning rate. These follow LR * hidden_size = 0.384, the muP-style
37
+ # rule that keeps update sizes comparable across widths. The anchor was
38
+ # checked by sweeping the 500k tier (1e-3 through 1.6e-2); 4e-3 won.
39
+ lr: float
40
+ batch_tokens: int # tokens per optimizer step (global)
41
+ micro_batch: int # sequences per forward pass, per GPU
42
+ # (halved automatically if CUDA OOMs)
43
+
44
+ @property
45
+ def head_dim(self) -> int:
46
+ return self.hidden_size // self.num_attention_heads
47
+
48
+ def param_count(self, vocab_size: int = VOCAB_SIZE) -> int:
49
+ h = self.hidden_size
50
+ kv = self.num_key_value_heads * self.head_dim
51
+ per_layer = (
52
+ h * h # q_proj
53
+ + h * kv # k_proj
54
+ + h * kv # v_proj
55
+ + h * h # o_proj
56
+ + 3 * h * self.intermediate_size # gate, up, down
57
+ + 2 * h # input_layernorm + post_attention_layernorm
58
+ )
59
+ return vocab_size * h + self.num_hidden_layers * per_layer + h
60
+
61
+ def to_hf_config_dict(self, vocab_size: int = VOCAB_SIZE) -> dict:
62
+ return {
63
+ "architectures": ["LlamaForCausalLM"],
64
+ "attention_bias": False,
65
+ "attention_dropout": 0.0,
66
+ "bos_token_id": 0,
67
+ "eos_token_id": 0,
68
+ "pad_token_id": 1,
69
+ "hidden_act": "silu",
70
+ "hidden_size": self.hidden_size,
71
+ "initializer_range": 0.041666666666666664,
72
+ "intermediate_size": self.intermediate_size,
73
+ "max_position_embeddings": MAX_POSITION_EMBEDDINGS,
74
+ "mlp_bias": False,
75
+ "model_type": "llama",
76
+ "num_attention_heads": self.num_attention_heads,
77
+ "num_hidden_layers": self.num_hidden_layers,
78
+ "num_key_value_heads": self.num_key_value_heads,
79
+ "pretraining_tp": 1,
80
+ "rms_norm_eps": RMS_NORM_EPS,
81
+ "rope_scaling": None,
82
+ "rope_theta": ROPE_THETA,
83
+ "tie_word_embeddings": True,
84
+ "torch_dtype": "float32",
85
+ "use_cache": True,
86
+ "vocab_size": vocab_size,
87
+ }
88
+
89
+ TIERS = {
90
+ "100k": Tier(
91
+ key="100k", label="NanoDex-100K",
92
+ hidden_size=32, num_hidden_layers=2,
93
+ num_attention_heads=2, num_key_value_heads=1,
94
+ intermediate_size=147,
95
+ lr=5e-3, batch_tokens=65536, micro_batch=128,
96
+ ),
97
+ "250k": Tier(
98
+ key="250k", label="NanoDex-250K",
99
+ hidden_size=48, num_hidden_layers=4,
100
+ num_attention_heads=4, num_key_value_heads=2,
101
+ intermediate_size=215,
102
+ lr=4.5e-3, batch_tokens=65536, micro_batch=128,
103
+ ),
104
+ "500k": Tier(
105
+ key="500k", label="NanoDex-500K",
106
+ hidden_size=80, num_hidden_layers=4,
107
+ num_attention_heads=5, num_key_value_heads=1,
108
+ intermediate_size=285,
109
+ lr=4e-3, batch_tokens=131072, micro_batch=256,
110
+ ),
111
+ "1m": Tier(
112
+ key="1m", label="NanoDex-1M",
113
+ hidden_size=96, num_hidden_layers=5,
114
+ num_attention_heads=6, num_key_value_heads=2,
115
+ intermediate_size=472,
116
+ lr=3e-3, batch_tokens=131072, micro_batch=256,
117
+ ),
118
+ "2m": Tier(
119
+ key="2m", label="NanoDex-2M",
120
+ hidden_size=128, num_hidden_layers=6,
121
+ num_attention_heads=8, num_key_value_heads=2,
122
+ intermediate_size=647,
123
+ lr=2.5e-3, batch_tokens=262144, micro_batch=256,
124
+ ),
125
+ "5m": Tier(
126
+ key="5m", label="NanoDex-5M",
127
+ hidden_size=192, num_hidden_layers=8,
128
+ num_attention_heads=8, num_key_value_heads=2,
129
+ intermediate_size=839,
130
+ lr=1.5e-3, batch_tokens=393216, micro_batch=128,
131
+ ),
132
+ "10m": Tier(
133
+ key="10m", label="NanoDex-10M",
134
+ hidden_size=256, num_hidden_layers=10,
135
+ num_attention_heads=8, num_key_value_heads=2,
136
+ intermediate_size=1020,
137
+ lr=1.2e-3, batch_tokens=524288, micro_batch=128,
138
+ ),
139
+ "25m": Tier(
140
+ key="25m", label="NanoDex-25M",
141
+ hidden_size=320, num_hidden_layers=12,
142
+ num_attention_heads=10, num_key_value_heads=2,
143
+ intermediate_size=1856,
144
+ lr=8e-4, batch_tokens=524288, micro_batch=64,
145
+ ),
146
+ "50m": Tier(
147
+ key="50m", label="NanoDex-50M",
148
+ hidden_size=448, num_hidden_layers=12,
149
+ num_attention_heads=8, num_key_value_heads=2,
150
+ intermediate_size=2669,
151
+ lr=6e-4, batch_tokens=524288, micro_batch=32,
152
+ ),
153
+ "100m": Tier(
154
+ key="100m", label="NanoDex-100M",
155
+ hidden_size=640, num_hidden_layers=21,
156
+ num_attention_heads=8, num_key_value_heads=2,
157
+ intermediate_size=1792,
158
+ lr=4e-4, batch_tokens=1048576, micro_batch=32,
159
+ ),
160
+ "gpt2-small": Tier(
161
+ key="gpt2-small", label="GPT-2 Small",
162
+ hidden_size=768, num_hidden_layers=12,
163
+ num_attention_heads=12, num_key_value_heads=12,
164
+ intermediate_size=3072,
165
+ lr=3e-4, batch_tokens=524288, micro_batch=16,
166
+ ),
167
+ }
168
+
169
+ TIER_ORDER = [
170
+ "100k",
171
+ "250k",
172
+ "500k",
173
+ "1m",
174
+ "2m",
175
+ "5m",
176
+ "10m",
177
+ "25m",
178
+ "50m",
179
+ "100m",
180
+ "gpt2-small"
181
+ ]
182
+
183
+ MIN_TOKENS = 1_000_000
184
+ MAX_TOKENS = 100_000_000_000
185
+
186
+ TOKEN_STOPS_M = [
187
+ 1, 5, 10, 25, 50, 100, 200, 300, 400, 500,
188
+ 750, 1000, 1500, 2000, 3000,
189
+ 4000, 5000, 6000, 7500, 8000,
190
+ 9000, 10000, 25000, 50000, 100000
191
+ ]
192
+
193
+ def tier_or_none(key):
194
+ """Tolerant lookup — old rows may reference tiers that no longer exist."""
195
+ return TIERS.get(key)
196
+
197
+
198
+ if __name__ == "__main__":
199
+ for k in TIER_ORDER:
200
+ t = TIERS[k]
201
+ p = t.param_count()
202
+ tpm = t.micro_batch * SEQ_LEN
203
+ accum = max(1, t.batch_tokens // tpm)
204
+ exact = "exact" if t.micro_batch * SEQ_LEN * accum == t.batch_tokens else "MISALIGNED"
205
+ print(f"{t.label:14s} {p:>10,d} h={t.hidden_size:3d} L={t.num_hidden_layers:2d} "
206
+ f"heads={t.num_attention_heads}/{t.num_key_value_heads} hd={t.head_dim} "
207
+ f"ffn={t.intermediate_size:3d} | micro={t.micro_batch} accum={accum} "
208
+ f"({t.batch_tokens:,} tok/step, {exact}) lr={t.lr:.1e}")
nanodex/data.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Shared fineweb-edu token cache.
3
+
4
+ Every job trains on the same corpus (HuggingFaceFW/fineweb-edu), so instead of
5
+ each run streaming and re-tokenizing the dataset, the Space builds ONE flat
6
+ uint16 token file in a background thread at startup and all trainers read
7
+ slices of it. 300M tokens x 2 bytes = 600 MB on disk, which comfortably fits
8
+ Spaces' ephemeral storage and makes the GPUs data-bound never again.
9
+
10
+ Documents are packed end-to-end separated by <|endoftext|> (id 0).
11
+ """
12
+
13
+ import os
14
+ import threading
15
+ import time
16
+
17
+ import numpy as np
18
+
19
+ from . import store
20
+ from .config import MAX_TOKENS
21
+ from .db import DATA_DIR
22
+
23
+ TOKENS_PATH = os.path.join(DATA_DIR, "fineweb_edu_tokens.u16")
24
+ META_PATH = TOKENS_PATH + ".meta"
25
+
26
+ DATASET_ID = "HuggingFaceFW/fineweb-edu"
27
+ DATASET_CONFIG = "sample-350BT"
28
+
29
+ # Build a little past the maximum a user can request so the last run never
30
+ # stalls on the writer. At 2 bytes per token a 1B-token budget is ~2 GB on
31
+ # disk, which is nothing next to the Space's ephemeral storage.
32
+ TARGET_TOKENS = MAX_TOKENS + 5_000_000
33
+
34
+ _state = {
35
+ "ready_tokens": 0,
36
+ "status": "idle", # idle|building|ready|error
37
+ "error": None,
38
+ "started": None,
39
+ "rate": 0.0,
40
+ }
41
+ _state_lock = threading.Lock()
42
+ _build_thread = None
43
+
44
+
45
+ def state():
46
+ with _state_lock:
47
+ return dict(_state)
48
+
49
+
50
+ def _set(**kw):
51
+ with _state_lock:
52
+ _state.update(kw)
53
+
54
+
55
+ def _load_meta():
56
+ if os.path.exists(META_PATH):
57
+ try:
58
+ with open(META_PATH) as f:
59
+ return int(f.read().strip())
60
+ except Exception:
61
+ return 0
62
+ return 0
63
+
64
+
65
+ def _save_meta(n):
66
+ tmp = META_PATH + ".tmp"
67
+ with open(tmp, "w") as f:
68
+ f.write(str(n))
69
+ os.replace(tmp, META_PATH)
70
+
71
+
72
+ def build_token_cache(tokenizer, log=print):
73
+ """Stream fineweb-edu, tokenize, append to the flat uint16 file."""
74
+ from datasets import load_dataset
75
+
76
+ # A previous container may have already paid for this. Downloading ~2.8 GB
77
+ # beats re-tokenizing 1.5B tokens by a wide margin.
78
+ have = _load_meta()
79
+ if have < TARGET_TOKENS:
80
+ restored = store.fetch_token_cache(TOKENS_PATH, META_PATH, log)
81
+ if restored > have:
82
+ have = restored
83
+ _set(ready_tokens=have)
84
+
85
+ if have >= TARGET_TOKENS:
86
+ _set(ready_tokens=have, status="ready")
87
+ log(f"[data] cache already complete: {have:,} tokens")
88
+ return
89
+
90
+ _set(status="building", started=time.time(), ready_tokens=have)
91
+ log(f"[data] building token cache ({have:,} / {TARGET_TOKENS:,})")
92
+
93
+ eot = 0 # <|endoftext|>
94
+ t0 = time.time()
95
+ written = have
96
+
97
+ # Truncate to the last known-good offset, then append.
98
+ mode = "r+b" if os.path.exists(TOKENS_PATH) else "wb"
99
+ f = open(TOKENS_PATH, mode)
100
+ f.seek(written * 2)
101
+ f.truncate()
102
+
103
+ try:
104
+ ds = load_dataset(DATASET_ID, name=DATASET_CONFIG,
105
+ split="train", streaming=True)
106
+ # Skip ahead deterministically if resuming a partial build.
107
+ buf_docs = []
108
+ BATCH = 512
109
+ for row in ds:
110
+ buf_docs.append(row["text"])
111
+ if len(buf_docs) < BATCH:
112
+ continue
113
+ encs = tokenizer.encode_batch(buf_docs)
114
+ buf_docs = []
115
+ flat = []
116
+ for e in encs:
117
+ flat.extend(e.ids)
118
+ flat.append(eot)
119
+ arr = np.asarray(flat, dtype=np.uint16)
120
+ f.write(arr.tobytes())
121
+ written += arr.size
122
+ if written % (1 << 22) < arr.size: # ~every 4M tokens
123
+ f.flush()
124
+ _save_meta(written)
125
+ _touch_lock()
126
+ dt = max(time.time() - t0, 1e-6)
127
+ _set(ready_tokens=written, rate=(written - have) / dt)
128
+ log(f"[data] {written:,} tokens "
129
+ f"({(written - have) / dt / 1e6:.2f}M tok/s)")
130
+ if written >= TARGET_TOKENS:
131
+ break
132
+ f.flush()
133
+ _save_meta(written)
134
+ _set(ready_tokens=written, status="ready")
135
+ log(f"[data] cache ready: {written:,} tokens in {time.time() - t0:.0f}s")
136
+ f.close()
137
+ store.save_token_cache(TOKENS_PATH, META_PATH, log)
138
+ except Exception as exc: # keep the Space alive; jobs will wait / use what exists
139
+ _save_meta(written)
140
+ _set(ready_tokens=written, status="error", error=repr(exc))
141
+ log(f"[data] ERROR: {exc!r} — the partial cache of {written:,} tokens is still usable")
142
+ finally:
143
+ f.close()
144
+
145
+
146
+ LOCK_PATH = TOKENS_PATH + ".lock"
147
+ LOCK_STALE_SECONDS = 150
148
+ _HEARTBEAT = 30
149
+
150
+
151
+ def _acquire_build_lock():
152
+ """
153
+ Only one process may append to the shared token file. A rolling deploy runs
154
+ two containers against the same volume for a while, and two appenders would
155
+ corrupt it. The lock is a file the holder keeps touching, so a crashed
156
+ holder's lock goes stale instead of blocking forever.
157
+ """
158
+ now = time.time()
159
+ try:
160
+ fd = os.open(LOCK_PATH, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
161
+ os.write(fd, str(os.getpid()).encode())
162
+ os.close(fd)
163
+ return True
164
+ except FileExistsError:
165
+ try:
166
+ age = now - os.path.getmtime(LOCK_PATH)
167
+ except OSError:
168
+ return False
169
+ if age < LOCK_STALE_SECONDS:
170
+ return False
171
+ # Stale: the previous holder is gone. Take it over.
172
+ try:
173
+ os.utime(LOCK_PATH, (now, now))
174
+ with open(LOCK_PATH, "w") as f:
175
+ f.write(str(os.getpid()))
176
+ return True
177
+ except OSError:
178
+ return False
179
+ except OSError:
180
+ return True # can't lock (read-only fs) — proceed anyway
181
+
182
+
183
+ def _touch_lock():
184
+ try:
185
+ os.utime(LOCK_PATH, None)
186
+ except OSError:
187
+ pass
188
+
189
+
190
+ def _release_lock():
191
+ try:
192
+ os.unlink(LOCK_PATH)
193
+ except OSError:
194
+ pass
195
+
196
+
197
+ def _guarded_build(tokenizer, log):
198
+ if not _acquire_build_lock():
199
+ log("[data] another process is building the token cache — "
200
+ "using it read-only")
201
+ _set(status="following")
202
+ # Track the other process's progress so jobs know what's available.
203
+ while True:
204
+ have = _load_meta()
205
+ _set(ready_tokens=have)
206
+ if have >= TARGET_TOKENS:
207
+ _set(status="ready")
208
+ return
209
+ try:
210
+ if time.time() - os.path.getmtime(LOCK_PATH) > LOCK_STALE_SECONDS:
211
+ break # holder died; take over below
212
+ except OSError:
213
+ break
214
+ time.sleep(10)
215
+ if not _acquire_build_lock():
216
+ _set(status="ready" if _load_meta() else "error")
217
+ return
218
+ try:
219
+ build_token_cache(tokenizer, log)
220
+ finally:
221
+ _release_lock()
222
+
223
+
224
+ def start_background_build(tokenizer, log=print):
225
+ global _build_thread
226
+ if _build_thread and _build_thread.is_alive():
227
+ return _build_thread
228
+ _build_thread = threading.Thread(
229
+ target=_guarded_build, args=(tokenizer, log),
230
+ name="token-cache", daemon=True)
231
+ _build_thread.start()
232
+ return _build_thread
233
+
234
+
235
+ def available_tokens():
236
+ with _state_lock:
237
+ return _state["ready_tokens"]
238
+
239
+
240
+ class TokenStream:
241
+ """
242
+ Sequential reader over the shared token file, yielding (B, T) int64
243
+ batches. Each job gets its own random start offset so two runs of the same
244
+ size do not see the identical token order.
245
+
246
+ The batch is used as BOTH input_ids and labels: LlamaForCausalLM shifts
247
+ labels internally, so handing it a pre-shifted target would train the model
248
+ to predict two tokens ahead while generation samples one ahead.
249
+ """
250
+
251
+ def __init__(self, seq_len, batch_size, seed=1337):
252
+ self.seq_len = seq_len
253
+ self.batch_size = batch_size
254
+ self.chunk = batch_size * seq_len
255
+ rng = np.random.default_rng(seed)
256
+ avail = max(available_tokens(), self.chunk + 1)
257
+ self.pos = int(rng.integers(0, max(avail - self.chunk, 1)))
258
+ self._mm = None
259
+ self._mm_len = 0
260
+
261
+ def _ensure(self):
262
+ n = available_tokens()
263
+ if self._mm is None or n > self._mm_len:
264
+ if n < self.chunk + 1:
265
+ return False
266
+ self._mm = np.memmap(TOKENS_PATH, dtype=np.uint16, mode="r", shape=(n,))
267
+ self._mm_len = n
268
+ return self._mm is not None
269
+
270
+ def next_batch(self):
271
+ """One (B, T) int64 batch. Pass it as input_ids AND labels."""
272
+ import torch
273
+ deadline = time.time() + 300
274
+ while not self._ensure():
275
+ if time.time() > deadline:
276
+ raise RuntimeError("token cache unavailable (timed out waiting for data)")
277
+ time.sleep(2)
278
+ if self.pos + self.chunk > self._mm_len:
279
+ self.pos = 0
280
+ buf = np.asarray(self._mm[self.pos:self.pos + self.chunk], dtype=np.int64)
281
+ self.pos += self.chunk
282
+ return torch.from_numpy(buf.reshape(self.batch_size, self.seq_len))
nanodex/db.py ADDED
@@ -0,0 +1,370 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SQLite-backed job store for the NanoDex training queue."""
2
+
3
+ import json
4
+ import os
5
+ import sqlite3
6
+ import threading
7
+ import time
8
+ import uuid
9
+
10
+ # Spaces persistent storage mounts at /data. Fall back to a local dir.
11
+ DATA_DIR = "/data" if os.path.isdir("/data") and os.access("/data", os.W_OK) else \
12
+ os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "runtime")
13
+ os.makedirs(DATA_DIR, exist_ok=True)
14
+ DB_PATH = os.path.join(DATA_DIR, "nanodex.db")
15
+ RUNS_DIR = os.path.join(DATA_DIR, "runs")
16
+ os.makedirs(RUNS_DIR, exist_ok=True)
17
+
18
+ _lock = threading.RLock()
19
+
20
+ SCHEMA = """
21
+ CREATE TABLE IF NOT EXISTS jobs (
22
+ id TEXT PRIMARY KEY,
23
+ username TEXT NOT NULL,
24
+ display_name TEXT,
25
+ avatar_url TEXT,
26
+ model_name TEXT NOT NULL,
27
+ tier TEXT NOT NULL,
28
+ n_params INTEGER NOT NULL,
29
+ target_tokens INTEGER NOT NULL,
30
+ status TEXT NOT NULL, -- queued|running|done|failed|cancelled
31
+ gpu_index INTEGER,
32
+ created_at REAL NOT NULL,
33
+ started_at REAL,
34
+ finished_at REAL,
35
+ tokens_seen INTEGER DEFAULT 0,
36
+ step INTEGER DEFAULT 0,
37
+ total_steps INTEGER DEFAULT 0,
38
+ loss REAL,
39
+ best_loss REAL,
40
+ lr REAL,
41
+ tok_per_s REAL,
42
+ eta_s REAL,
43
+ error TEXT,
44
+ pushed_repo TEXT,
45
+ seed INTEGER DEFAULT 1337,
46
+ live_step INTEGER,
47
+ live_at REAL
48
+ );
49
+ CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status, created_at);
50
+ CREATE INDEX IF NOT EXISTS idx_jobs_user ON jobs(username, created_at DESC);
51
+
52
+ CREATE TABLE IF NOT EXISTS metrics (
53
+ job_id TEXT NOT NULL,
54
+ step INTEGER NOT NULL,
55
+ tokens INTEGER NOT NULL,
56
+ loss REAL NOT NULL,
57
+ lr REAL,
58
+ t REAL NOT NULL,
59
+ PRIMARY KEY (job_id, step)
60
+ );
61
+
62
+ CREATE TABLE IF NOT EXISTS logs (
63
+ job_id TEXT NOT NULL,
64
+ t REAL NOT NULL,
65
+ line TEXT NOT NULL
66
+ );
67
+ CREATE INDEX IF NOT EXISTS idx_logs_job ON logs(job_id, t);
68
+ """
69
+
70
+
71
+ def _conn():
72
+ c = sqlite3.connect(DB_PATH, timeout=10, check_same_thread=False)
73
+ c.row_factory = sqlite3.Row
74
+ c.execute("PRAGMA journal_mode=WAL")
75
+ c.execute("PRAGMA busy_timeout=10000")
76
+ return c
77
+
78
+
79
+ # Columns added after the first release; applied to existing databases on boot.
80
+ MIGRATIONS = [
81
+ "ALTER TABLE jobs ADD COLUMN live_step INTEGER",
82
+ "ALTER TABLE jobs ADD COLUMN live_at REAL",
83
+ ]
84
+
85
+
86
+ def init():
87
+ with _lock, _conn() as c:
88
+ c.executescript(SCHEMA)
89
+ for stmt in MIGRATIONS:
90
+ try:
91
+ c.execute(stmt)
92
+ except sqlite3.OperationalError:
93
+ pass # already present
94
+
95
+ # Any job left 'running' belongs to a container that is gone. This is a
96
+ # write, so it can contend with another process holding the database (a
97
+ # rolling deploy briefly runs two containers against the same volume) —
98
+ # never let that stall the boot.
99
+ try:
100
+ with _lock, _conn() as c:
101
+ c.execute(
102
+ "UPDATE jobs SET status='failed', "
103
+ "error='interrupted (the Space restarted)', finished_at=? "
104
+ "WHERE status='running'",
105
+ (time.time(),),
106
+ )
107
+ except sqlite3.Error as exc:
108
+ print(f"[db] could not reap orphaned runs: {exc!r}", flush=True)
109
+
110
+
111
+ def create_job(username, display_name, avatar_url, model_name, tier,
112
+ n_params, target_tokens, total_steps, seed=1337):
113
+ job_id = uuid.uuid4().hex[:12]
114
+ with _lock, _conn() as c:
115
+ c.execute(
116
+ "INSERT INTO jobs (id, username, display_name, avatar_url, model_name, tier,"
117
+ " n_params, target_tokens, status, created_at, total_steps, seed)"
118
+ " VALUES (?,?,?,?,?,?,?,?, 'queued', ?, ?, ?)",
119
+ (job_id, username, display_name, avatar_url, model_name, tier,
120
+ n_params, target_tokens, time.time(), total_steps, seed),
121
+ )
122
+ return job_id
123
+
124
+
125
+ def update_job(job_id, **fields):
126
+ if not fields:
127
+ return
128
+ cols = ", ".join(f"{k}=?" for k in fields)
129
+ with _lock, _conn() as c:
130
+ c.execute(f"UPDATE jobs SET {cols} WHERE id=?", (*fields.values(), job_id))
131
+
132
+
133
+ def get_job(job_id):
134
+ with _lock, _conn() as c:
135
+ r = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()
136
+ return dict(r) if r else None
137
+
138
+
139
+ def claim_next_queued(gpu_index):
140
+ """Atomically move the oldest queued job to running. Returns the job or None."""
141
+ with _lock, _conn() as c:
142
+ r = c.execute(
143
+ "SELECT * FROM jobs WHERE status='queued' ORDER BY created_at LIMIT 1"
144
+ ).fetchone()
145
+ if not r:
146
+ return None
147
+ n = c.execute(
148
+ "UPDATE jobs SET status='running', gpu_index=?, started_at=? "
149
+ "WHERE id=? AND status='queued'",
150
+ (gpu_index, time.time(), r["id"]),
151
+ ).rowcount
152
+ if n == 0:
153
+ return None
154
+ return dict(c.execute("SELECT * FROM jobs WHERE id=?", (r["id"],)).fetchone())
155
+
156
+
157
+ def list_jobs(status=None, username=None, limit=100):
158
+ q = "SELECT * FROM jobs"
159
+ where, args = [], []
160
+ if status:
161
+ where.append("status IN (%s)" % ",".join("?" * len(status)))
162
+ args += list(status)
163
+ if username:
164
+ where.append("username=?")
165
+ args.append(username)
166
+ if where:
167
+ q += " WHERE " + " AND ".join(where)
168
+ q += " ORDER BY created_at DESC LIMIT ?"
169
+ args.append(limit)
170
+ with _lock, _conn() as c:
171
+ return [dict(r) for r in c.execute(q, args).fetchall()]
172
+
173
+
174
+ def queue_position(job_id):
175
+ with _lock, _conn() as c:
176
+ r = c.execute("SELECT created_at FROM jobs WHERE id=?", (job_id,)).fetchone()
177
+ if not r:
178
+ return None
179
+ return c.execute(
180
+ "SELECT COUNT(*) FROM jobs WHERE status='queued' AND created_at < ?",
181
+ (r["created_at"],),
182
+ ).fetchone()[0] + 1
183
+
184
+
185
+ def user_active_count(username):
186
+ with _lock, _conn() as c:
187
+ return c.execute(
188
+ "SELECT COUNT(*) FROM jobs WHERE username=? AND status IN ('queued','running')",
189
+ (username,),
190
+ ).fetchone()[0]
191
+
192
+
193
+ def add_metric(job_id, step, tokens, loss, lr, t=None):
194
+ with _lock, _conn() as c:
195
+ c.execute(
196
+ "INSERT OR REPLACE INTO metrics (job_id, step, tokens, loss, lr, t) VALUES (?,?,?,?,?,?)",
197
+ (job_id, step, tokens, loss, lr, t or time.time()),
198
+ )
199
+
200
+
201
+ def get_metrics(job_id, limit=3000):
202
+ with _lock, _conn() as c:
203
+ return [dict(r) for r in c.execute(
204
+ "SELECT step, tokens, loss, lr FROM metrics WHERE job_id=? ORDER BY step LIMIT ?",
205
+ (job_id, limit),
206
+ ).fetchall()]
207
+
208
+
209
+ def add_log(job_id, line):
210
+ with _lock, _conn() as c:
211
+ c.execute("INSERT INTO logs (job_id, t, line) VALUES (?,?,?)",
212
+ (job_id, time.time(), line))
213
+
214
+
215
+ def get_logs(job_id, limit=300):
216
+ with _lock, _conn() as c:
217
+ rows = c.execute(
218
+ "SELECT t, line FROM logs WHERE job_id=? ORDER BY t DESC LIMIT ?",
219
+ (job_id, limit),
220
+ ).fetchall()
221
+ return [dict(r) for r in reversed(rows)]
222
+
223
+
224
+ # ---------------------------------------------------------- community ------
225
+ _SORTS = {
226
+ "recent": "COALESCE(finished_at, created_at) DESC",
227
+ "loss": "best_loss ASC",
228
+ "tokens": "tokens_seen DESC",
229
+ "params": "n_params DESC, tokens_seen DESC",
230
+ }
231
+
232
+
233
+ def list_public(tier=None, sort="recent", limit=60, offset=0, username=None):
234
+ """Finished runs from everyone — the site's shared shelf of models."""
235
+ q = ("SELECT * FROM jobs WHERE status='done' AND tokens_seen > 0 "
236
+ "AND best_loss IS NOT NULL")
237
+ args = []
238
+ if tier:
239
+ q += " AND tier=?"
240
+ args.append(tier)
241
+ if username:
242
+ q += " AND username=?"
243
+ args.append(username)
244
+ q += f" ORDER BY {_SORTS.get(sort, _SORTS['recent'])} LIMIT ? OFFSET ?"
245
+ args += [limit, offset]
246
+ with _lock, _conn() as c:
247
+ return [dict(r) for r in c.execute(q, args).fetchall()]
248
+
249
+
250
+ def list_active_public(limit=40):
251
+ """Everything training or queued right now, from everyone."""
252
+ with _lock, _conn() as c:
253
+ return [dict(r) for r in c.execute(
254
+ "SELECT * FROM jobs WHERE status IN ('running','queued') "
255
+ "ORDER BY CASE status WHEN 'running' THEN 0 ELSE 1 END, created_at "
256
+ "LIMIT ?", (limit,)).fetchall()]
257
+
258
+
259
+ def leaderboard(limit_per_tier=3):
260
+ """Lowest loss per tier. Same tokenizer, data and context for everyone,
261
+ so the numbers are directly comparable."""
262
+ out = {}
263
+ with _lock, _conn() as c:
264
+ tiers = [r[0] for r in c.execute(
265
+ "SELECT DISTINCT tier FROM jobs WHERE status='done' "
266
+ "AND best_loss IS NOT NULL").fetchall()]
267
+ for t in tiers:
268
+ out[t] = [dict(r) for r in c.execute(
269
+ "SELECT * FROM jobs WHERE status='done' AND tier=? "
270
+ "AND best_loss IS NOT NULL ORDER BY best_loss ASC LIMIT ?",
271
+ (t, limit_per_tier)).fetchall()]
272
+ return out
273
+
274
+
275
+ def user_profile(username):
276
+ """Public totals for one person, plus the avatar they last signed in with."""
277
+ with _lock, _conn() as c:
278
+ row = c.execute(
279
+ "SELECT COUNT(*) AS runs,"
280
+ " SUM(CASE WHEN status='done' THEN 1 ELSE 0 END) AS done,"
281
+ " SUM(CASE WHEN status IN ('queued','running') THEN 1 ELSE 0 END) AS active,"
282
+ " COALESCE(SUM(tokens_seen),0) AS tokens,"
283
+ " MIN(CASE WHEN status='done' THEN best_loss END) AS best_loss,"
284
+ " SUM(CASE WHEN pushed_repo IS NOT NULL THEN 1 ELSE 0 END) AS published,"
285
+ " MIN(created_at) AS first_seen, MAX(created_at) AS last_seen"
286
+ " FROM jobs WHERE username=?", (username,)).fetchone()
287
+ if not row or not row["runs"]:
288
+ return None
289
+ meta = c.execute(
290
+ "SELECT display_name, avatar_url FROM jobs WHERE username=? "
291
+ "ORDER BY created_at DESC LIMIT 1", (username,)).fetchone()
292
+ params = c.execute(
293
+ "SELECT COALESCE(SUM(n_params),0) FROM jobs WHERE username=? "
294
+ "AND status='done'", (username,)).fetchone()[0]
295
+ d = dict(row)
296
+ d.update(username=username, params=params,
297
+ display_name=(meta["display_name"] if meta else None) or username,
298
+ avatar_url=meta["avatar_url"] if meta else None)
299
+ return d
300
+
301
+
302
+ def top_trainers(limit=12):
303
+ with _lock, _conn() as c:
304
+ return [dict(r) for r in c.execute(
305
+ "SELECT username, display_name, avatar_url,"
306
+ " COUNT(*) AS runs,"
307
+ " SUM(CASE WHEN status='done' THEN 1 ELSE 0 END) AS done,"
308
+ " COALESCE(SUM(tokens_seen),0) AS tokens,"
309
+ " MIN(CASE WHEN status='done' THEN best_loss END) AS best_loss"
310
+ " FROM jobs GROUP BY username"
311
+ " ORDER BY done DESC, tokens DESC LIMIT ?", (limit,)).fetchall()]
312
+
313
+
314
+ # -------------------------------------------------- durable round-trip -----
315
+ def export_run(job_id):
316
+ """Everything about one run, in one JSON-serialisable dict."""
317
+ j = get_job(job_id)
318
+ if not j:
319
+ return None
320
+ return {"job": j, "metrics": get_metrics(job_id, 5000),
321
+ "logs": get_logs(job_id, 400), "schema": 1}
322
+
323
+
324
+ def import_run(payload):
325
+ """Insert a run exported earlier. Existing rows are left untouched."""
326
+ j = payload.get("job") or {}
327
+ jid = j.get("id")
328
+ if not jid:
329
+ return False
330
+ cols = [
331
+ "id", "username", "display_name", "avatar_url", "model_name", "tier",
332
+ "n_params", "target_tokens", "status", "gpu_index", "created_at",
333
+ "started_at", "finished_at", "tokens_seen", "step", "total_steps",
334
+ "loss", "best_loss", "lr", "tok_per_s", "eta_s", "error",
335
+ "pushed_repo", "seed", "live_step", "live_at",
336
+ ]
337
+ vals = [j.get(c) for c in cols]
338
+ with _lock, _conn() as c:
339
+ if c.execute("SELECT 1 FROM jobs WHERE id=?", (jid,)).fetchone():
340
+ return False
341
+ c.execute(
342
+ f"INSERT INTO jobs ({','.join(cols)}) "
343
+ f"VALUES ({','.join('?' * len(cols))})", vals)
344
+ for m in payload.get("metrics", []):
345
+ c.execute("INSERT OR REPLACE INTO metrics "
346
+ "(job_id, step, tokens, loss, lr, t) VALUES (?,?,?,?,?,?)",
347
+ (jid, m.get("step"), m.get("tokens"), m.get("loss"),
348
+ m.get("lr"), j.get("finished_at") or time.time()))
349
+ for l in payload.get("logs", []):
350
+ c.execute("INSERT INTO logs (job_id, t, line) VALUES (?,?,?)",
351
+ (jid, l.get("t") or time.time(), l.get("line", "")))
352
+ return True
353
+
354
+
355
+ def known_job_ids():
356
+ with _lock, _conn() as c:
357
+ return {r[0] for r in c.execute("SELECT id FROM jobs").fetchall()}
358
+
359
+
360
+ def stats():
361
+ with _lock, _conn() as c:
362
+ row = c.execute(
363
+ "SELECT "
364
+ " (SELECT COUNT(*) FROM jobs WHERE status='queued') AS queued,"
365
+ " (SELECT COUNT(*) FROM jobs WHERE status='running') AS running,"
366
+ " (SELECT COUNT(*) FROM jobs WHERE status='done') AS done,"
367
+ " (SELECT COUNT(DISTINCT username) FROM jobs) AS users,"
368
+ " (SELECT COALESCE(SUM(tokens_seen),0) FROM jobs) AS tokens"
369
+ ).fetchone()
370
+ return dict(row)
nanodex/estimate.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Throughput estimates, calibrated from this Space's own completed runs."""
2
+
3
+ import sqlite3
4
+
5
+ from . import db
6
+ from .config import TIERS, SEQ_LEN, VOCAB_SIZE
7
+
8
+ # Two tiers have now been measured on a single L4, and that is enough to fit
9
+ # the cost model instead of guessing at it:
10
+ #
11
+ # seconds/token = HEAD + body_flops / RATE
12
+ #
13
+ # HEAD is the tied lm_head matmul plus the softmax/cross-entropy over the
14
+ # 2,048-token vocabulary. It depends on the number of tokens, not on the model
15
+ # size — 40% of the 500k tier's step time, 21% of the 1M's. RATE is the
16
+ # effective throughput of the transformer stack itself.
17
+ #
18
+ # Solving the two measurements gives HEAD ~= 0.54 us/token and
19
+ # RATE ~= 3.3 TFLOP/s. The larger tiers get a width correction because wider
20
+ # matmuls use the GPU better; that correction is the least certain part here,
21
+ # so the 5M and 8M numbers carry real uncertainty until runs of those sizes
22
+ # finish and replace the prior outright.
23
+ MEASURED_ANCHORS = { # tier -> tokens/s on one L4
24
+ "500k": 737_000.0,
25
+ "1m": 382_000.0,
26
+ }
27
+ _WIDTH_EXP = 0.6 # body efficiency vs hidden size
28
+ CPU_PENALTY = 60.0
29
+
30
+
31
+ def _body_flops_per_token(t):
32
+ """FLOPs excluding the lm_head: the transformer stack and attention."""
33
+ non_emb = t.param_count() - VOCAB_SIZE * t.hidden_size
34
+ return 6 * non_emb + 6 * t.num_hidden_layers * SEQ_LEN * t.hidden_size
35
+
36
+
37
+ def _fit():
38
+ """Solve HEAD and RATE from the measured anchors."""
39
+ keys = [k for k in MEASURED_ANCHORS if k in TIERS]
40
+ if len(keys) < 2:
41
+ return None
42
+ keys.sort(key=lambda k: _body_flops_per_token(TIERS[k]))
43
+ lo, hi = keys[0], keys[-1]
44
+ f_lo, f_hi = (_body_flops_per_token(TIERS[k]) for k in (lo, hi))
45
+ t_lo, t_hi = (1.0 / MEASURED_ANCHORS[k] for k in (lo, hi))
46
+ if f_hi <= f_lo or t_hi <= t_lo:
47
+ return None
48
+ rate = (f_hi - f_lo) / (t_hi - t_lo)
49
+ head = t_lo - f_lo / rate
50
+ if head <= 0 or rate <= 0:
51
+ return None
52
+ return head, rate, TIERS[hi].hidden_size
53
+
54
+
55
+ _FIT = _fit()
56
+
57
+
58
+ def _prior(tier_key):
59
+ if tier_key in MEASURED_ANCHORS:
60
+ return MEASURED_ANCHORS[tier_key]
61
+ t = TIERS[tier_key]
62
+ if not _FIT:
63
+ return 300_000.0
64
+ head, rate, ref_h = _FIT
65
+ rate = rate * (t.hidden_size / ref_h) ** _WIDTH_EXP
66
+ return 1.0 / (head + _body_flops_per_token(t) / rate)
67
+
68
+
69
+ PRIOR_TOK_S = {k: _prior(k) for k in TIERS}
70
+
71
+
72
+ def measured_tok_s(tier_key):
73
+ try:
74
+ with db._lock, db._conn() as c:
75
+ rows = c.execute(
76
+ "SELECT tok_per_s FROM jobs WHERE tier=? AND status='done' "
77
+ "AND tok_per_s > 0 ORDER BY finished_at DESC LIMIT 9",
78
+ (tier_key,),
79
+ ).fetchall()
80
+ vals = sorted(r[0] for r in rows)
81
+ if len(vals) >= 2:
82
+ return vals[len(vals) // 2]
83
+ except sqlite3.Error:
84
+ pass
85
+ return None
86
+
87
+
88
+ def tok_s_for(tier_key, on_gpu=True):
89
+ m = measured_tok_s(tier_key)
90
+ if m:
91
+ return m
92
+ base = PRIOR_TOK_S[tier_key]
93
+ return base if on_gpu else base / CPU_PENALTY
94
+
95
+
96
+ def eta_seconds(tier_key, target_tokens, on_gpu=True):
97
+ return target_tokens / max(tok_s_for(tier_key, on_gpu), 1.0)
98
+
99
+
100
+ def human_time(sec):
101
+ sec = max(sec, 0)
102
+ if sec < 90:
103
+ return f"{sec:.0f}s"
104
+ if sec < 5400:
105
+ return f"{sec / 60:.0f} min"
106
+ return f"{sec / 3600:.1f} h"
107
+
108
+
109
+ def summary(tier_key, target_tokens, on_gpu=True):
110
+ tier = TIERS[tier_key]
111
+ n = tier.param_count()
112
+ steps = max(1, target_tokens // tier.batch_tokens)
113
+ eta = eta_seconds(tier_key, target_tokens, on_gpu)
114
+ ratio = target_tokens / n
115
+ return {
116
+ "n_params": n, "steps": steps, "eta_s": eta, "ratio": ratio,
117
+ "calibrated": measured_tok_s(tier_key) is not None,
118
+ }
nanodex/hub.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Push a finished HyperDex run to the user's own Hugging Face account."""
2
+
3
+ import json
4
+ import os
5
+
6
+ from huggingface_hub import HfApi
7
+
8
+ from .config import TIERS
9
+
10
+
11
+ def model_card(job, run_meta, repo_id):
12
+ tier = TIERS[job["tier"]]
13
+ loss = run_meta.get("final_loss")
14
+ import math
15
+ ppl = math.exp(min(loss, 20)) if loss else None
16
+ return f"""---
17
+ license: odc-by
18
+ datasets:
19
+ - HuggingFaceFW/fineweb-edu
20
+ language:
21
+ - en
22
+ library_name: transformers
23
+ pipeline_tag: text-generation
24
+ tags:
25
+ - nanodex
26
+ - tiny-lm
27
+ - pretrained-from-scratch
28
+ ---
29
+
30
+ # {repo_id.split('/')[-1]}
31
+
32
+ A **{run_meta['n_params']:,}-parameter** decoder-only language model pre-trained
33
+ **from scratch** on [fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu),
34
+ using the [HyperDex Trainer](https://hugging-science-hyperdex-trainer.hf.space/) Space.
35
+
36
+ ## Architecture
37
+
38
+ A standard `LlamaForCausalLM` decoder-only transformer — SiLU MLP, RMSNorm,
39
+ rotary position embeddings, grouped-query attention, tied embeddings, no biases —
40
+ scaled down in width and depth to fit the parameter budget.
41
+
42
+ | | |
43
+ |---|---|
44
+ | Parameters | {run_meta['n_params']:,} |
45
+ | Hidden size | {tier.hidden_size} |
46
+ | Layers | {tier.num_hidden_layers} |
47
+ | Attention heads | {tier.num_attention_heads} (KV: {tier.num_key_value_heads}) |
48
+ | FFN size | {tier.intermediate_size} |
49
+ | Context length | {run_meta['seq_len']} |
50
+ | Vocab | 2,048 (custom BPE trained on fineweb-edu) |
51
+
52
+ ## Training
53
+
54
+ | | |
55
+ |---|---|
56
+ | Tokens seen | {run_meta['tokens_seen']:,} |
57
+ | Steps | {run_meta['steps']:,} |
58
+ | Tokens / step | {run_meta['batch_tokens']:,} |
59
+ | Optimizer | {run_meta['optimizer']} |
60
+ | LR schedule | {run_meta['schedule']} (peak {run_meta['peak_lr']:.0e}) |
61
+ | Final loss | {loss:.4f}{f" (ppl {ppl:.1f})" if ppl else ""} |
62
+ | Wall time | {run_meta['wall_time_s'] / 60:.1f} min |
63
+ | Trained by | [@{job['username']}](https://huggingface.co/{job['username']}) |
64
+
65
+ ## Usage
66
+
67
+ ```python
68
+ from transformers import AutoModelForCausalLM, AutoTokenizer
69
+
70
+ tok = AutoTokenizer.from_pretrained("{repo_id}")
71
+ model = AutoModelForCausalLM.from_pretrained("{repo_id}")
72
+
73
+ ids = tok("The mitochondria is", return_tensors="pt").input_ids
74
+ print(tok.decode(model.generate(ids, max_new_tokens=60, do_sample=True,
75
+ temperature=0.8, top_k=50)[0]))
76
+ ```
77
+
78
+ ## Caveats
79
+
80
+ This is a **small-scale research artifact**. At this parameter count and token
81
+ budget the model learns word shapes, common collocations and a little syntax —
82
+ it is not a useful assistant and its output is not factual. It exists to make
83
+ "pre-train a transformer from scratch" something you can actually watch happen.
84
+ """
85
+
86
+
87
+ def push_run(out_dir, job, token, repo_name, private=False):
88
+ api = HfApi(token=token)
89
+ who = api.whoami()
90
+ owner = who["name"]
91
+ repo_id = f"{owner}/{repo_name}"
92
+
93
+ api.create_repo(repo_id, repo_type="model", private=private, exist_ok=True)
94
+
95
+ with open(os.path.join(out_dir, "training_run.json")) as f:
96
+ run_meta = json.load(f)
97
+
98
+ with open(os.path.join(out_dir, "README.md"), "w") as f:
99
+ f.write(model_card(job, run_meta, repo_id))
100
+
101
+ api.upload_folder(
102
+ folder_path=out_dir,
103
+ repo_id=repo_id,
104
+ repo_type="model",
105
+ # `live/` holds the mid-training snapshots that power the in-page
106
+ # playground; they have no business in the published model.
107
+ ignore_patterns=["live/*", "live", "*.tmp", "*.lock"],
108
+ commit_message=f"HyperDex {job['tier']} · {run_meta['tokens_seen']:,} fineweb-edu "
109
+ f"tokens · loss {run_meta['final_loss']:.4f}",
110
+ )
111
+ return repo_id
nanodex/store.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Durable storage on a Hugging Face bucket.
3
+
4
+ A Space's container filesystem dies on every rebuild, restart and garbage
5
+ collection, which would otherwise mean losing every model, every loss curve and
6
+ the whole 1.5B-token cache each time the code changes. This module mirrors the
7
+ three things worth keeping into a bucket:
8
+
9
+ cache/tokens.u16 + cache/tokens.meta the shared fineweb-edu token cache
10
+ runs/<job_id>/meta.json the job row, its metrics and logs
11
+ runs/<job_id>/<weights, tokenizer, …> the finished checkpoint
12
+
13
+ The design is deliberately append-only and keyed by job id: a finished run is
14
+ written once and never mutated. That removes every concurrency problem a rolling
15
+ deploy would otherwise create, because two containers never contend for the same
16
+ object — unlike syncing one shared SQLite file, where the last writer would
17
+ silently clobber the other.
18
+
19
+ On boot the local database is rebuilt from the run metadata in the bucket.
20
+ Checkpoints are pulled on demand, the first time someone opens or publishes a
21
+ model, so startup stays fast.
22
+
23
+ Runs that were in flight when a container died are NOT recoverable — the
24
+ optimizer state lives only in that process. They come back marked interrupted.
25
+ """
26
+
27
+ import json
28
+ import os
29
+ import shutil
30
+ import threading
31
+ import time
32
+
33
+ BUCKET_ID = os.environ.get("NANODEX_BUCKET", "hugging-science/nanodex-store")
34
+ CACHE_PREFIX = "cache/"
35
+ RUNS_PREFIX = "runs/"
36
+
37
+ # Files worth keeping from a finished run directory.
38
+ CHECKPOINT_FILES = (
39
+ "config.json",
40
+ "generation_config.json",
41
+ "model.safetensors",
42
+ "tokenizer.json",
43
+ "tokenizer_config.json",
44
+ "special_tokens_map.json",
45
+ "training_run.json",
46
+ )
47
+
48
+ _lock = threading.Lock()
49
+ _state = {"enabled": None, "error": None, "restored": 0, "last_error_at": 0}
50
+
51
+
52
+ def _api():
53
+ from huggingface_hub import HfApi
54
+ tok = os.environ.get("HF_TOKEN") or os.environ.get("NANODEX_HF_TOKEN")
55
+ return HfApi(token=tok) if tok else HfApi()
56
+
57
+
58
+ def state():
59
+ return dict(_state)
60
+
61
+
62
+ def available(log=print):
63
+ """True when the bucket is reachable and writable by this Space."""
64
+ if _state["enabled"] is not None:
65
+ return _state["enabled"]
66
+ with _lock:
67
+ if _state["enabled"] is not None:
68
+ return _state["enabled"]
69
+ try:
70
+ _api().bucket_info(BUCKET_ID)
71
+ _state["enabled"] = True
72
+ log(f"[store] bucket ready: {BUCKET_ID}")
73
+ except Exception as exc:
74
+ _state.update(enabled=False, error=repr(exc))
75
+ log(f"[store] bucket unavailable ({exc!r}) — "
76
+ "running with container-local storage only")
77
+ return _state["enabled"]
78
+
79
+
80
+ def _note_error(what, exc, log=print):
81
+ _state.update(error=f"{what}: {exc!r}", last_error_at=time.time())
82
+ log(f"[store] {what} failed: {exc!r}")
83
+
84
+
85
+ # --------------------------------------------------------------- uploads ----
86
+ def put_files(pairs, log=print):
87
+ """pairs: [(local_path, remote_path)] — uploaded in one batch."""
88
+ if not pairs or not available(log):
89
+ return False
90
+ try:
91
+ _api().batch_bucket_files(
92
+ BUCKET_ID, add=[(str(l), r) for l, r in pairs])
93
+ return True
94
+ except Exception as exc:
95
+ _note_error(f"upload of {len(pairs)} file(s)", exc, log)
96
+ return False
97
+
98
+
99
+ def put_json(obj, remote_path, log=print):
100
+ if not available(log):
101
+ return False
102
+ try:
103
+ payload = json.dumps(obj, indent=2).encode()
104
+ _api().batch_bucket_files(BUCKET_ID, add=[(payload, remote_path)])
105
+ return True
106
+ except Exception as exc:
107
+ _note_error(f"upload of {remote_path}", exc, log)
108
+ return False
109
+
110
+
111
+ def save_run(job_id, run_dir, meta, log=print):
112
+ """Persist one finished run: its metadata and its checkpoint files."""
113
+ if not available(log):
114
+ return False
115
+ ok = put_json(meta, f"{RUNS_PREFIX}{job_id}/meta.json", log)
116
+ pairs = []
117
+ for fn in CHECKPOINT_FILES:
118
+ p = os.path.join(run_dir, fn)
119
+ if os.path.exists(p):
120
+ pairs.append((p, f"{RUNS_PREFIX}{job_id}/{fn}"))
121
+ if pairs:
122
+ ok = put_files(pairs, log) and ok
123
+ log(f"[store] saved run {job_id} ({len(pairs)} files)")
124
+ return ok
125
+
126
+
127
+ def save_token_cache(tokens_path, meta_path, log=print):
128
+ """Upload the shared token cache so the next container skips tokenizing."""
129
+ if not available(log) or not os.path.exists(tokens_path):
130
+ return False
131
+ log("[store] uploading token cache (this happens once)")
132
+ t0 = time.time()
133
+ ok = put_files([(tokens_path, CACHE_PREFIX + "tokens.u16"),
134
+ (meta_path, CACHE_PREFIX + "tokens.meta")], log)
135
+ if ok:
136
+ log(f"[store] token cache uploaded in {time.time() - t0:.0f}s")
137
+ return ok
138
+
139
+
140
+ # ------------------------------------------------------------- downloads ----
141
+ def _remote_size(path, log=print):
142
+ try:
143
+ return _api().get_bucket_file_metadata(BUCKET_ID, path).size
144
+ except Exception:
145
+ return None
146
+
147
+
148
+ def fetch_token_cache(tokens_path, meta_path, log=print):
149
+ """
150
+ Pull the token cache if the bucket has one at least as complete as what is
151
+ on disk. Returns the number of tokens now available locally, or 0.
152
+ """
153
+ if not available(log):
154
+ return 0
155
+ try:
156
+ raw = _api().download_bucket_files(
157
+ BUCKET_ID,
158
+ [(CACHE_PREFIX + "tokens.meta", meta_path + ".remote")],
159
+ raise_on_missing_files=True)
160
+ except Exception:
161
+ return 0 # nothing stored yet — normal first boot
162
+ try:
163
+ with open(meta_path + ".remote") as f:
164
+ remote_tokens = int(f.read().strip())
165
+ except Exception:
166
+ return 0
167
+ finally:
168
+ try:
169
+ os.unlink(meta_path + ".remote")
170
+ except OSError:
171
+ pass
172
+
173
+ local_tokens = 0
174
+ if os.path.exists(meta_path):
175
+ try:
176
+ with open(meta_path) as f:
177
+ local_tokens = int(f.read().strip())
178
+ except Exception:
179
+ local_tokens = 0
180
+ if remote_tokens <= local_tokens:
181
+ return local_tokens
182
+
183
+ log(f"[store] downloading cached {remote_tokens:,} tokens "
184
+ f"(beats re-tokenizing from scratch)")
185
+ t0 = time.time()
186
+ try:
187
+ _api().download_bucket_files(
188
+ BUCKET_ID, [(CACHE_PREFIX + "tokens.u16", tokens_path)],
189
+ raise_on_missing_files=True)
190
+ with open(meta_path, "w") as f:
191
+ f.write(str(remote_tokens))
192
+ log(f"[store] token cache restored in {time.time() - t0:.0f}s")
193
+ return remote_tokens
194
+ except Exception as exc:
195
+ _note_error("token cache download", exc, log)
196
+ return local_tokens
197
+
198
+
199
+ def list_runs(log=print):
200
+ """Job ids that have metadata stored in the bucket."""
201
+ if not available(log):
202
+ return []
203
+ try:
204
+ ids = set()
205
+ for entry in _api().list_bucket_tree(BUCKET_ID, prefix=RUNS_PREFIX,
206
+ recursive=True):
207
+ path = getattr(entry, "path", None) or getattr(entry, "name", "")
208
+ rest = str(path)[len(RUNS_PREFIX):] if str(path).startswith(RUNS_PREFIX) else ""
209
+ if "/" in rest:
210
+ ids.add(rest.split("/", 1)[0])
211
+ return sorted(ids)
212
+ except Exception as exc:
213
+ _note_error("listing runs", exc, log)
214
+ return []
215
+
216
+
217
+ def fetch_run_meta(job_id, log=print):
218
+ if not available(log):
219
+ return None
220
+ tmp = f"/tmp/nanodex-meta-{job_id}.json"
221
+ try:
222
+ _api().download_bucket_files(
223
+ BUCKET_ID, [(f"{RUNS_PREFIX}{job_id}/meta.json", tmp)],
224
+ raise_on_missing_files=True)
225
+ with open(tmp) as f:
226
+ return json.load(f)
227
+ except Exception:
228
+ return None
229
+ finally:
230
+ try:
231
+ os.unlink(tmp)
232
+ except OSError:
233
+ pass
234
+
235
+
236
+ def fetch_checkpoint(job_id, run_dir, log=print):
237
+ """
238
+ Restore a finished run's weights into run_dir. Called lazily, the first
239
+ time somebody opens the playground or publishes after a restart.
240
+ """
241
+ if not available(log):
242
+ return False
243
+ if os.path.exists(os.path.join(run_dir, "model.safetensors")):
244
+ return True
245
+ os.makedirs(run_dir, exist_ok=True)
246
+ wanted = [(f"{RUNS_PREFIX}{job_id}/{fn}", os.path.join(run_dir, fn))
247
+ for fn in CHECKPOINT_FILES]
248
+ try:
249
+ _api().download_bucket_files(BUCKET_ID, wanted,
250
+ raise_on_missing_files=False)
251
+ except Exception as exc:
252
+ _note_error(f"checkpoint download for {job_id}", exc, log)
253
+ return False
254
+ ok = os.path.exists(os.path.join(run_dir, "model.safetensors"))
255
+ if ok:
256
+ log(f"[store] restored checkpoint {job_id}")
257
+ else:
258
+ shutil.rmtree(run_dir, ignore_errors=True)
259
+ return ok
nanodex/trainer.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pre-training loop for a single NanoDex run."""
2
+
3
+ import json
4
+ import math
5
+ import os
6
+ import shutil
7
+ import time
8
+
9
+ from . import db, store
10
+ from .config import TIERS, SEQ_LEN, VOCAB_SIZE
11
+
12
+ # Metric cadence adapts to run length so short runs still get a real curve.
13
+ def _cadence(total):
14
+ return max(1, min(25, total // 200))
15
+
16
+
17
+ def total_steps_for(tier_key, target_tokens):
18
+ tier = TIERS[tier_key]
19
+ return max(1, int(target_tokens // tier.batch_tokens))
20
+
21
+
22
+ def _lr_at(step, total, base_lr, warmup_frac=0.02, min_frac=0.1):
23
+ warmup = max(1, int(total * warmup_frac))
24
+ if step < warmup:
25
+ return base_lr * (step + 1) / warmup
26
+ prog = (step - warmup) / max(1, total - warmup)
27
+ prog = min(max(prog, 0.0), 1.0)
28
+ return base_lr * (min_frac + (1 - min_frac) * 0.5 * (1 + math.cos(math.pi * prog)))
29
+
30
+
31
+ LIVE_SNAPSHOTS = 40 # snapshots spread over a run
32
+ LIVE_MIN_SECONDS = 25 # ...but never more often than this
33
+
34
+
35
+ def live_dir(job_id):
36
+ return os.path.join(db.RUNS_DIR, job_id, "live")
37
+
38
+
39
+ def _write_live_checkpoint(model, job_id, step, tokenizer_dir, first=False):
40
+ """
41
+ Dump the current weights so the run page can generate text from the model
42
+ mid-training. Written to a temp file and renamed, so a reader never sees a
43
+ half-written checkpoint.
44
+ """
45
+ import torch
46
+ from safetensors.torch import save_file
47
+
48
+ d = live_dir(job_id)
49
+ os.makedirs(d, exist_ok=True)
50
+ if first:
51
+ cfg = model.config.to_dict()
52
+ cfg["use_cache"] = True
53
+ cfg["torch_dtype"] = "float32"
54
+ with open(os.path.join(d, "config.json"), "w") as f:
55
+ json.dump(cfg, f, indent=2)
56
+ with open(os.path.join(d, "generation_config.json"), "w") as f:
57
+ json.dump({"bos_token_id": 0, "eos_token_id": 0,
58
+ "pad_token_id": 1}, f, indent=2)
59
+ for fn in ("tokenizer.json", "tokenizer_config.json",
60
+ "special_tokens_map.json"):
61
+ src = os.path.join(tokenizer_dir, fn)
62
+ if os.path.exists(src):
63
+ shutil.copy(src, os.path.join(d, fn))
64
+
65
+ sd = {k: v.detach().to("cpu", torch.float32).contiguous()
66
+ for k, v in model.state_dict().items()
67
+ if not k.endswith("lm_head.weight")} # tied to the embedding
68
+ tmp = os.path.join(d, "model.safetensors.tmp")
69
+ save_file(sd, tmp, metadata={"format": "pt", "step": str(step)})
70
+ os.replace(tmp, os.path.join(d, "model.safetensors"))
71
+ db.update_job(job_id, live_step=step, live_at=time.time())
72
+
73
+
74
+ def build_model(tier_key, vocab_size=VOCAB_SIZE):
75
+ import torch
76
+ try:
77
+ # transformers >= 5 dropped the top-level re-exports
78
+ from transformers.models.llama import LlamaConfig, LlamaForCausalLM
79
+ except ImportError:
80
+ from transformers import LlamaConfig, LlamaForCausalLM
81
+
82
+ tier = TIERS[tier_key]
83
+ cfg = LlamaConfig(**tier.to_hf_config_dict(vocab_size))
84
+ cfg.use_cache = False
85
+ model = LlamaForCausalLM(cfg)
86
+ return model
87
+
88
+
89
+ def run_training(job, device, tokenizer_dir, cancel_flag, log):
90
+ """
91
+ Execute one job to completion. `cancel_flag` is a callable returning True
92
+ when the user asked to stop. Returns the output directory.
93
+ """
94
+ import torch
95
+ from .data import TokenStream
96
+
97
+ job_id = job["id"]
98
+ tier = TIERS[job["tier"]]
99
+ target_tokens = int(job["target_tokens"])
100
+ total = total_steps_for(job["tier"], target_tokens)
101
+ seq_len = SEQ_LEN
102
+
103
+ use_cuda = device.startswith("cuda")
104
+ torch.manual_seed(int(job.get("seed") or 1337))
105
+
106
+ model = build_model(job["tier"]).to(device)
107
+ n_params = sum(p.numel() for p in model.parameters())
108
+ log(f"model: {tier.label} · {n_params:,} parameters · device={device}")
109
+
110
+ if use_cuda:
111
+ torch.backends.cuda.matmul.allow_tf32 = True
112
+ torch.backends.cudnn.allow_tf32 = True
113
+
114
+ micro = tier.micro_batch if use_cuda else max(4, tier.micro_batch // 32)
115
+ tokens_per_micro = micro * seq_len
116
+ accum = max(1, tier.batch_tokens // tokens_per_micro)
117
+ log(f"batch: {micro} seq x {seq_len} tok x {accum} accum "
118
+ f"= {micro * seq_len * accum:,} tokens/step · {total:,} steps")
119
+
120
+ opt = torch.optim.AdamW(
121
+ model.parameters(), lr=tier.lr, betas=(0.9, 0.95),
122
+ weight_decay=0.1, eps=1e-8, fused=use_cuda,
123
+ )
124
+ stream = TokenStream(seq_len, micro, seed=int(job.get("seed") or 1337))
125
+ amp_dtype = torch.bfloat16 if use_cuda else torch.float32
126
+
127
+ out_dir = os.path.join(db.RUNS_DIR, job_id)
128
+ os.makedirs(out_dir, exist_ok=True)
129
+
130
+ every = _cadence(total)
131
+ live_every = max(1, total // LIVE_SNAPSHOTS)
132
+ last_live = 0.0
133
+ live_started = False
134
+ model.train()
135
+ tokens_seen = 0
136
+ best = float("inf")
137
+ t_start = time.time()
138
+ t_win = time.time()
139
+ tok_win = 0
140
+
141
+ for step in range(total):
142
+ if cancel_flag():
143
+ log("cancelled by the user")
144
+ db.update_job(job_id, status="cancelled", finished_at=time.time())
145
+ return None
146
+
147
+ lr = _lr_at(step, total, tier.lr)
148
+ for g in opt.param_groups:
149
+ g["lr"] = lr
150
+
151
+ while True:
152
+ opt.zero_grad(set_to_none=True)
153
+ loss_acc = 0.0
154
+ step_tokens = 0
155
+ try:
156
+ for _ in range(accum):
157
+ x = stream.next_batch().to(device, non_blocking=True)
158
+ with torch.autocast(device_type="cuda" if use_cuda else "cpu",
159
+ dtype=amp_dtype, enabled=use_cuda):
160
+ # labels == input_ids: the model does the causal shift
161
+ # itself. Passing a pre-shifted target would train it to
162
+ # predict two tokens ahead of where generation reads.
163
+ out = model(input_ids=x, labels=x)
164
+ loss = out.loss / accum
165
+ loss.backward()
166
+ loss_acc += loss.item()
167
+ step_tokens += x.numel()
168
+ break
169
+ except torch.cuda.OutOfMemoryError:
170
+ if micro <= 4:
171
+ raise
172
+ micro = max(4, micro // 2)
173
+ accum = max(1, tier.batch_tokens // (micro * seq_len))
174
+ opt.zero_grad(set_to_none=True)
175
+ torch.cuda.empty_cache()
176
+ stream = TokenStream(seq_len, micro,
177
+ seed=int(job.get("seed") or 1337))
178
+ log(f"out of GPU memory — retrying with {micro} seq x {accum} accum")
179
+
180
+ tokens_seen += step_tokens
181
+ tok_win += step_tokens
182
+
183
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
184
+ opt.step()
185
+
186
+ best = min(best, loss_acc)
187
+
188
+ # Snapshot for the in-page playground.
189
+ if (step % live_every == 0 or step == total - 1) and \
190
+ (time.time() - last_live > LIVE_MIN_SECONDS or step == total - 1):
191
+ try:
192
+ _write_live_checkpoint(model, job_id, step, tokenizer_dir,
193
+ first=not live_started)
194
+ live_started = True
195
+ last_live = time.time()
196
+ except Exception as exc:
197
+ log(f"live checkpoint skipped: {exc!r}")
198
+
199
+ if step % every == 0 or step == total - 1:
200
+ now = time.time()
201
+ rate = tok_win / max(now - t_win, 1e-6)
202
+ t_win, tok_win = now, 0
203
+ eta = (target_tokens - tokens_seen) / max(rate, 1.0)
204
+ db.add_metric(job_id, step, tokens_seen, loss_acc, lr)
205
+ db.update_job(
206
+ job_id, step=step, tokens_seen=tokens_seen, loss=loss_acc,
207
+ best_loss=best, lr=lr, tok_per_s=rate, eta_s=eta,
208
+ )
209
+ if step % (every * 4) == 0 or step == total - 1:
210
+ log(f"step {step:>6,}/{total:,} · loss {loss_acc:.4f} · "
211
+ f"ppl {math.exp(min(loss_acc, 20)):.1f} · lr {lr:.2e} · "
212
+ f"{rate / 1e3:.0f}k tok/s · ETA {eta / 60:.1f} min")
213
+
214
+ # ---- save ----
215
+ log("saving checkpoint...")
216
+ model.config.use_cache = True
217
+ model = model.float().cpu()
218
+ model.save_pretrained(out_dir, safe_serialization=True)
219
+ for fn in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"):
220
+ src = os.path.join(tokenizer_dir, fn)
221
+ if os.path.exists(src):
222
+ shutil.copy(src, os.path.join(out_dir, fn))
223
+ with open(os.path.join(out_dir, "generation_config.json"), "w") as f:
224
+ json.dump({"bos_token_id": 0, "eos_token_id": 0, "pad_token_id": 1,
225
+ "_from_model_config": True}, f, indent=2)
226
+ with open(os.path.join(out_dir, "training_run.json"), "w") as f:
227
+ json.dump({
228
+ "job_id": job_id, "tier": job["tier"], "label": tier.label,
229
+ "n_params": n_params, "target_tokens": target_tokens,
230
+ "tokens_seen": tokens_seen, "steps": total, "seq_len": seq_len,
231
+ "final_loss": loss_acc, "best_loss": best,
232
+ "peak_lr": tier.lr, "batch_tokens": tier.batch_tokens,
233
+ "optimizer": "AdamW(0.9, 0.95) wd=0.1 clip=1.0",
234
+ "schedule": "warmup 2% + cosine to 10%",
235
+ "dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)",
236
+ "architecture": "LlamaForCausalLM (SiLU, RMSNorm, RoPE, GQA, tied embeddings)",
237
+ "wall_time_s": time.time() - t_start,
238
+ "trained_by": job["username"],
239
+ }, f, indent=2)
240
+
241
+ db.update_job(job_id, status="done", finished_at=time.time(),
242
+ tokens_seen=tokens_seen, loss=loss_acc, best_loss=best,
243
+ step=total, eta_s=0)
244
+
245
+ # Mirror the finished run to the bucket so it survives the next rebuild.
246
+ try:
247
+ payload = db.export_run(job_id)
248
+ if payload:
249
+ store.save_run(job_id, out_dir, payload, log)
250
+ except Exception as exc:
251
+ log(f"could not archive the run to durable storage: {exc!r}")
252
+ log(f"training finished in {(time.time() - t_start) / 60:.1f} min · "
253
+ f"final loss {loss_acc:.4f}")
254
+ return out_dir
nanodex/worker.py ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """One worker thread per GPU, pulling jobs off the shared queue."""
2
+
3
+ import threading
4
+ import time
5
+ import traceback
6
+
7
+ from . import db
8
+ from .trainer import run_training
9
+
10
+ _cancel = set()
11
+ _cancel_lock = threading.Lock()
12
+ _workers = []
13
+
14
+
15
+ def request_cancel(job_id):
16
+ with _cancel_lock:
17
+ _cancel.add(job_id)
18
+ j = db.get_job(job_id)
19
+ if j and j["status"] == "queued":
20
+ db.update_job(job_id, status="cancelled", finished_at=time.time())
21
+ return True
22
+ return bool(j)
23
+
24
+
25
+ def _is_cancelled(job_id):
26
+ with _cancel_lock:
27
+ return job_id in _cancel
28
+
29
+
30
+ _devices_cache = None
31
+
32
+
33
+ def devices():
34
+ """Never raises and never blocks twice — CUDA probing is done once."""
35
+ global _devices_cache
36
+ if _devices_cache is not None:
37
+ return _devices_cache
38
+ found = ["cpu"]
39
+ try:
40
+ import torch
41
+ if torch.cuda.is_available() and torch.cuda.device_count() > 0:
42
+ found = [f"cuda:{i}" for i in range(torch.cuda.device_count())]
43
+ except Exception as exc:
44
+ print(f"[worker] no CUDA ({exc!r}) — falling back to CPU", flush=True)
45
+ _devices_cache = found
46
+ return found
47
+
48
+
49
+ def _loop(gpu_index, device, tokenizer_dir):
50
+ while True:
51
+ try:
52
+ job = db.claim_next_queued(gpu_index)
53
+ except Exception:
54
+ job = None
55
+ if not job:
56
+ time.sleep(2)
57
+ continue
58
+
59
+ jid = job["id"]
60
+
61
+ def log(msg, _jid=jid, _dev=device):
62
+ print(f"[{_dev}][{_jid}] {msg}", flush=True)
63
+ try:
64
+ db.add_log(_jid, msg)
65
+ except Exception:
66
+ pass
67
+
68
+ # Don't start on a barely-filled cache, or a 300M-token run would
69
+ # loop over the same few million tokens hundreds of times.
70
+ try:
71
+ from . import data as ndata
72
+ need = min(int(job["target_tokens"]), ndata.TARGET_TOKENS)
73
+ last_note = 0.0
74
+ while ndata.available_tokens() < need and ndata.state()["status"] == "building":
75
+ if time.time() - last_note > 60:
76
+ log(f"waiting for the fineweb-edu cache to reach "
77
+ f"{need / 1e6:.0f}M tokens "
78
+ f"({ndata.available_tokens() / 1e6:.0f}M ready so far — "
79
+ f"this is a one-time cost per container)")
80
+ last_note = time.time()
81
+ if _is_cancelled(jid):
82
+ break
83
+ time.sleep(5)
84
+ except Exception:
85
+ pass
86
+
87
+ if _is_cancelled(jid):
88
+ db.update_job(jid, status="cancelled", finished_at=time.time())
89
+ with _cancel_lock:
90
+ _cancel.discard(jid)
91
+ continue
92
+
93
+ log(f"starting on {device}")
94
+ try:
95
+ run_training(job, device, tokenizer_dir, lambda: _is_cancelled(jid), log)
96
+ except Exception as exc:
97
+ tb = traceback.format_exc(limit=6)
98
+ print(tb, flush=True)
99
+ log(f"FAILED: {exc!r}")
100
+ db.update_job(jid, status="failed", error=str(exc)[:500],
101
+ finished_at=time.time())
102
+ finally:
103
+ with _cancel_lock:
104
+ _cancel.discard(jid)
105
+ try:
106
+ import torch
107
+ if device.startswith("cuda"):
108
+ torch.cuda.empty_cache()
109
+ except Exception:
110
+ pass
111
+
112
+
113
+ def start_workers(tokenizer_dir):
114
+ global _workers
115
+ if _workers:
116
+ return _workers
117
+ for i, dev in enumerate(devices()):
118
+ if dev.startswith("cuda"):
119
+ try:
120
+ import torch
121
+ name = torch.cuda.get_device_name(i)
122
+ gb = torch.cuda.get_device_properties(i).total_memory / 1024**3
123
+ print(f"[worker] {dev}: {name} ({gb:.0f} GB)", flush=True)
124
+ except Exception:
125
+ pass
126
+ t = threading.Thread(target=_loop, args=(i, dev, tokenizer_dir),
127
+ name=f"worker-{dev}", daemon=True)
128
+ t.start()
129
+ _workers.append(t)
130
+ print(f"[worker] started on {dev}", flush=True)
131
+ return _workers
132
+
133
+
134
+ def worker_info():
135
+ devs = devices()
136
+ running = {j["gpu_index"]: j for j in db.list_jobs(status=["running"], limit=64)}
137
+ return [{"index": i, "device": d, "job": running.get(i)} for i, d in enumerate(devs)]
requirements.txt ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi>=0.115
2
+ uvicorn[standard]>=0.32
3
+ jinja2>=3.1
4
+ python-multipart>=0.0.12
5
+ itsdangerous>=2.2
6
+ authlib>=1.3
7
+ httpx>=0.27
8
+
9
+ torch>=2.6
10
+ transformers>=4.48
11
+ datasets>=3.2
12
+ tokenizers>=0.21
13
+ safetensors>=0.4
14
+ huggingface_hub>=0.27
15
+ numpy>=1.26
tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|pad|>"
5
+ }
tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|pad|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ }
19
+ },
20
+ "bos_token": "<|endoftext|>",
21
+ "eos_token": "<|endoftext|>",
22
+ "pad_token": "<|pad|>",
23
+ "unk_token": null,
24
+ "clean_up_tokenization_spaces": false,
25
+ "model_max_length": 512,
26
+ "tokenizer_class": "PreTrainedTokenizerFast"
27
+ }
web/static/app.css ADDED
@@ -0,0 +1,351 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ :root{
2
+ --bg:#080b0f; --bg-2:#0d1117; --panel:#11161d; --panel-2:#161d26;
3
+ --line:#1e2733; --line-2:#2a3441;
4
+ --fg:#e7edf5; --fg-2:#9aa8ba; --fg-3:#64748b;
5
+ --accent:#2ee6a8; --accent-2:#19b884; --accent-dim:#2ee6a81a;
6
+ --blue:#5aa9ff; --violet:#a78bfa; --amber:#fbbf24; --red:#fb7185;
7
+ --r:14px; --r-sm:9px;
8
+ --mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace;
9
+ --sans:-apple-system,BlinkMacSystemFont,"Inter","Segoe UI",Roboto,sans-serif;
10
+ }
11
+ *{box-sizing:border-box}
12
+ html,body{margin:0;padding:0}
13
+ body{background:var(--bg);color:var(--fg);font-family:var(--sans);
14
+ font-size:15px;line-height:1.6;-webkit-font-smoothing:antialiased;
15
+ min-height:100vh;display:flex;flex-direction:column}
16
+ a{color:inherit;text-decoration:none}
17
+ h1,h2,h3,h4{margin:0;line-height:1.2;letter-spacing:-.022em;font-weight:650}
18
+ ::selection{background:var(--accent);color:#04140e}
19
+
20
+ /* ---------- layout ---------- */
21
+ .wrap{width:100%;max-width:1140px;margin:0 auto;padding:0 24px}
22
+ main{flex:1;padding:36px 0 72px}
23
+ .grid{display:grid;gap:16px}
24
+ .g2{grid-template-columns:repeat(2,1fr)}
25
+ .g3{grid-template-columns:repeat(3,1fr)}
26
+ .g4{grid-template-columns:repeat(4,1fr)}
27
+ @media(max-width:880px){.g2,.g3,.g4{grid-template-columns:1fr}}
28
+
29
+ /* ---------- nav ---------- */
30
+ nav{position:sticky;top:0;z-index:50;background:rgba(8,11,15,.82);
31
+ backdrop-filter:blur(14px);border-bottom:1px solid var(--line)}
32
+ nav .wrap{display:flex;align-items:center;gap:8px;height:60px}
33
+ .brand{display:flex;align-items:center;gap:9px;font-weight:700;font-size:16.5px;
34
+ letter-spacing:-.03em;margin-right:14px}
35
+ .brand .dot{width:9px;height:9px;border-radius:50%;background:var(--accent);
36
+ box-shadow:0 0 12px var(--accent)}
37
+ .navlinks{display:flex;gap:2px;flex:1}
38
+ .navlinks a{padding:7px 13px;border-radius:8px;color:var(--fg-2);font-size:14px;
39
+ font-weight:500;transition:.15s}
40
+ .navlinks a:hover{color:var(--fg);background:var(--panel)}
41
+ .navlinks a.on{color:var(--accent);background:var(--accent-dim)}
42
+ .avatar{width:30px;height:30px;border-radius:50%;border:1px solid var(--line-2);
43
+ object-fit:cover;background:var(--panel-2)}
44
+ .usermenu{display:flex;align-items:center;gap:9px;padding:4px 11px 4px 4px;
45
+ border:1px solid var(--line);border-radius:99px;font-size:13.5px;font-weight:500}
46
+ .usermenu:hover{border-color:var(--line-2);background:var(--panel)}
47
+ @media(max-width:760px){.navlinks a{padding:7px 9px;font-size:13px}
48
+ .usermenu span{display:none}}
49
+
50
+ /* ---------- buttons ---------- */
51
+ .btn{display:inline-flex;align-items:center;justify-content:center;gap:8px;
52
+ padding:10px 18px;border-radius:var(--r-sm);border:1px solid var(--line-2);
53
+ background:var(--panel);color:var(--fg);font:inherit;font-size:14px;
54
+ font-weight:560;cursor:pointer;transition:.15s;white-space:nowrap}
55
+ .btn:hover{background:var(--panel-2);border-color:#374151}
56
+ .btn:disabled{opacity:.4;cursor:not-allowed}
57
+ .btn.primary{background:var(--accent);border-color:var(--accent);color:#04140e}
58
+ .btn.primary:hover{background:#48f0b8;border-color:#48f0b8}
59
+ .btn.ghost{background:transparent;border-color:transparent;color:var(--fg-2)}
60
+ .btn.ghost:hover{color:var(--fg);background:var(--panel)}
61
+ .btn.danger{border-color:#4c1d24;color:var(--red);background:transparent}
62
+ .btn.danger:hover{background:#2a1015}
63
+ .btn.lg{padding:13px 26px;font-size:15px}
64
+ .btn.sm{padding:6px 12px;font-size:13px}
65
+
66
+ /* ---------- cards ---------- */
67
+ .card{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);
68
+ padding:20px}
69
+ .card.pad-lg{padding:26px}
70
+ .card h3{font-size:16px;margin-bottom:3px}
71
+ .sub{color:var(--fg-2);font-size:13.5px}
72
+ .muted{color:var(--fg-3)}
73
+ .mono{font-family:var(--mono);font-variant-numeric:tabular-nums}
74
+ .eyebrow{font-size:11px;text-transform:uppercase;letter-spacing:.11em;
75
+ color:var(--fg-3);font-weight:650}
76
+ .sect-h{display:flex;align-items:baseline;justify-content:space-between;
77
+ gap:12px;margin:0 0 14px}
78
+ .sect-h h2{font-size:19px}
79
+
80
+ /* ---------- chips ---------- */
81
+ .chip{display:inline-flex;align-items:center;gap:6px;font-size:11.5px;
82
+ font-weight:650;padding:3px 10px;border-radius:99px;letter-spacing:.02em;
83
+ color:var(--c);background:color-mix(in srgb,var(--c) 13%,transparent);
84
+ border:1px solid color-mix(in srgb,var(--c) 30%,transparent);white-space:nowrap}
85
+ .chip i{width:5px;height:5px;border-radius:50%;background:var(--c);flex:none}
86
+ .chip.queued{--c:var(--violet)} .chip.running{--c:var(--accent)}
87
+ .chip.done{--c:var(--blue)} .chip.failed{--c:var(--red)}
88
+ .chip.cancelled{--c:#64748b}
89
+ .chip.running i{animation:pulse 1.4s ease-in-out infinite}
90
+ @keyframes pulse{0%,100%{opacity:1}50%{opacity:.25}}
91
+
92
+ /* ---------- progress ---------- */
93
+ .bar{height:6px;border-radius:99px;background:var(--line);overflow:hidden}
94
+ .bar>i{display:block;height:100%;border-radius:99px;
95
+ background:linear-gradient(90deg,var(--accent-2),var(--accent));
96
+ transition:width .5s ease}
97
+ .bar.thin{height:4px}
98
+
99
+ /* ---------- stat tiles ---------- */
100
+ .stat{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);
101
+ padding:16px 18px}
102
+ .stat b{display:block;font-size:25px;font-weight:680;letter-spacing:-.03em;
103
+ font-variant-numeric:tabular-nums;line-height:1.15}
104
+ .stat span{font-size:11px;text-transform:uppercase;letter-spacing:.1em;
105
+ color:var(--fg-3);font-weight:650}
106
+ .stat small{display:block;color:var(--fg-3);font-size:12px;margin-top:2px}
107
+
108
+ /* ---------- run rows ---------- */
109
+ .runs{display:flex;flex-direction:column;gap:9px}
110
+ .run{display:block;background:var(--panel);border:1px solid var(--line);
111
+ border-radius:var(--r);padding:15px 18px;transition:.15s}
112
+ .run:hover{border-color:var(--line-2);background:var(--panel-2)}
113
+ .run-top{display:flex;align-items:center;gap:11px;margin-bottom:4px}
114
+ .run-top h4{font-size:15px;font-weight:620;flex:1;min-width:0;
115
+ overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
116
+ .run-meta{display:flex;gap:14px;flex-wrap:wrap;font-size:12.5px;color:var(--fg-3);
117
+ font-variant-numeric:tabular-nums}
118
+ .run-meta b{color:var(--fg-2);font-weight:600}
119
+ .run .bar{margin-top:11px}
120
+
121
+ /* ---------- forms ---------- */
122
+ label.fl{display:block;font-size:12.5px;font-weight:620;color:var(--fg-2);
123
+ margin-bottom:7px}
124
+ input[type=text],input[type=number],textarea,select{width:100%;padding:11px 14px;
125
+ background:var(--bg-2);border:1px solid var(--line-2);border-radius:var(--r-sm);
126
+ color:var(--fg);font:inherit;font-size:14.5px;transition:.15s}
127
+ input:focus,textarea:focus,select:focus{outline:none;border-color:var(--accent);
128
+ box-shadow:0 0 0 3px var(--accent-dim)}
129
+ input::placeholder,textarea::placeholder{color:#4b5563}
130
+ textarea{resize:vertical;font-family:var(--mono);font-size:13.5px;line-height:1.65}
131
+ .hint{font-size:12.5px;color:var(--fg-3);margin-top:7px}
132
+
133
+ input[type=range]{-webkit-appearance:none;appearance:none;width:100%;height:6px;
134
+ background:var(--line);border-radius:99px;padding:0;border:none;outline:none}
135
+ input[type=range]::-webkit-slider-thumb{-webkit-appearance:none;width:20px;height:20px;
136
+ border-radius:50%;background:var(--accent);cursor:grab;border:3px solid var(--bg);
137
+ box-shadow:0 0 0 1px var(--accent)}
138
+ input[type=range]::-moz-range-thumb{width:20px;height:20px;border-radius:50%;
139
+ background:var(--accent);cursor:grab;border:3px solid var(--bg);box-shadow:0 0 0 1px var(--accent)}
140
+
141
+ /* ---------- pick cards ---------- */
142
+ .picks{display:grid;grid-template-columns:repeat(4,1fr);gap:11px}
143
+ @media(max-width:820px){.picks{grid-template-columns:repeat(2,1fr)}}
144
+ .pick{background:var(--bg-2);border:1.5px solid var(--line-2);border-radius:var(--r);
145
+ padding:16px;cursor:pointer;transition:.15s;text-align:left;color:inherit;
146
+ font:inherit;display:block}
147
+ .pick:hover{border-color:#3b4b5e;background:var(--panel-2)}
148
+ .pick.on{border-color:var(--accent);background:var(--accent-dim);
149
+ box-shadow:0 0 0 3px var(--accent-dim)}
150
+ .pick b{display:block;font-size:15.5px;font-weight:660;letter-spacing:-.02em}
151
+ .pick .p-n{font-family:var(--mono);font-size:12.5px;color:var(--accent);
152
+ margin:4px 0 8px;font-variant-numeric:tabular-nums}
153
+ .pick .p-d{font-size:12px;color:var(--fg-3);line-height:1.55}
154
+
155
+ /* ---------- steps ---------- */
156
+ .steps{display:flex;gap:0;margin-bottom:28px;border-bottom:1px solid var(--line)}
157
+ .stp{flex:1;padding:0 0 13px;border-bottom:2px solid transparent;
158
+ margin-bottom:-1px;color:var(--fg-3);transition:.2s}
159
+ .stp.on{color:var(--accent);border-bottom-color:var(--accent)}
160
+ .stp.ok{color:var(--fg-2);border-bottom-color:var(--line-2)}
161
+ .stp .n{font-family:var(--mono);font-size:11px;letter-spacing:.1em;
162
+ text-transform:uppercase;font-weight:700}
163
+ .stp .t{font-size:14px;font-weight:600;margin-top:2px}
164
+ @media(max-width:700px){.stp .t{font-size:12px}}
165
+ .panelstep{display:none} .panelstep.on{display:block;animation:fade .25s ease}
166
+ @keyframes fade{from{opacity:0;transform:translateY(6px)}to{opacity:1;transform:none}}
167
+
168
+ /* ---------- chart ---------- */
169
+ .chart{width:100%;height:280px;display:block}
170
+ .chart .gl{stroke:var(--line);stroke-width:1}
171
+ .chart .ax{fill:var(--fg-3);font-size:10.5px;font-family:var(--mono)}
172
+ .chart .ln{fill:none;stroke:var(--accent);stroke-width:2;
173
+ stroke-linejoin:round;stroke-linecap:round}
174
+ .chart .ar{fill:url(#lg)}
175
+
176
+ /* ---------- logs ---------- */
177
+ .logs{background:#05080b;border:1px solid var(--line);border-radius:var(--r-sm);
178
+ padding:14px 16px;font-family:var(--mono);font-size:12.2px;line-height:1.75;
179
+ max-height:340px;overflow:auto;color:#9fb3c8;white-space:pre-wrap;word-break:break-word}
180
+ .logs .t{color:#3f5063;margin-right:9px}
181
+
182
+ /* ---------- table ---------- */
183
+ table{width:100%;border-collapse:collapse;font-size:13.5px}
184
+ th{text-align:left;font-size:11px;text-transform:uppercase;letter-spacing:.09em;
185
+ color:var(--fg-3);font-weight:650;padding:0 14px 10px;border-bottom:1px solid var(--line)}
186
+ td{padding:13px 14px;border-bottom:1px solid var(--line);
187
+ font-variant-numeric:tabular-nums}
188
+ tbody tr:hover{background:var(--panel)}
189
+ tbody tr:last-child td{border-bottom:none}
190
+
191
+ /* ---------- landing ---------- */
192
+ .hero{padding:76px 0 56px;text-align:center;position:relative;overflow:hidden}
193
+ .hero::before{content:"";position:absolute;inset:-45% 15% auto;height:420px;
194
+ background:radial-gradient(ellipse at center,rgba(46,230,168,.13),transparent 65%);
195
+ pointer-events:none}
196
+ .hero h1{font-size:clamp(35px,6.4vw,60px);font-weight:720;letter-spacing:-.045em;
197
+ position:relative;line-height:1.05}
198
+ .hero h1 em{font-style:normal;
199
+ background:linear-gradient(100deg,var(--accent),var(--blue));
200
+ -webkit-background-clip:text;background-clip:text;-webkit-text-fill-color:transparent}
201
+ .hero p{font-size:clamp(15.5px,2.2vw,19px);color:var(--fg-2);max-width:640px;
202
+ margin:20px auto 0;position:relative}
203
+ .hero .cta{display:flex;gap:12px;justify-content:center;margin-top:34px;
204
+ position:relative;flex-wrap:wrap}
205
+ .tag{display:inline-flex;align-items:center;gap:8px;padding:5px 14px;
206
+ border:1px solid var(--line-2);border-radius:99px;font-size:12.5px;
207
+ color:var(--fg-2);margin-bottom:24px;position:relative;background:var(--panel)}
208
+ .tag b{color:var(--accent);font-weight:640}
209
+
210
+ .step-n{width:26px;height:26px;border-radius:8px;background:var(--accent-dim);
211
+ color:var(--accent);display:grid;place-items:center;font-family:var(--mono);
212
+ font-size:12px;font-weight:700;margin-bottom:11px}
213
+
214
+ /* ---------- misc ---------- */
215
+ .empty{padding:52px 24px;text-align:center;color:var(--fg-3);
216
+ border:1px dashed var(--line-2);border-radius:var(--r);font-size:14px}
217
+ .toast{position:fixed;bottom:22px;left:50%;transform:translateX(-50%) translateY(80px);
218
+ background:var(--panel-2);border:1px solid var(--line-2);border-radius:var(--r-sm);
219
+ padding:13px 20px;font-size:14px;z-index:200;opacity:0;transition:.28s;
220
+ box-shadow:0 12px 40px rgba(0,0,0,.55);max-width:min(560px,90vw)}
221
+ .toast.on{opacity:1;transform:translateX(-50%)}
222
+ .toast.err{border-color:#4c1d24;color:#ffc4cc}
223
+ .toast.ok{border-color:var(--accent-2)}
224
+ .gpu{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);
225
+ padding:15px 17px}
226
+ .gpu.busy{border-color:color-mix(in srgb,var(--accent) 34%,var(--line))}
227
+ .gpu-h{display:flex;justify-content:space-between;align-items:baseline;
228
+ font-size:13.5px;font-weight:640;margin-bottom:9px}
229
+ .gpu-h span{font-family:var(--mono);font-size:11px;color:var(--fg-3)}
230
+ footer{border-top:1px solid var(--line);padding:26px 0;color:var(--fg-3);
231
+ font-size:13px}
232
+ footer .wrap{display:flex;justify-content:space-between;gap:16px;flex-wrap:wrap}
233
+ footer a{color:var(--fg-2)} footer a:hover{color:var(--accent)}
234
+ .kv{display:flex;justify-content:space-between;gap:14px;padding:9px 0;
235
+ border-bottom:1px solid var(--line);font-size:13.5px}
236
+ .kv:last-child{border-bottom:none}
237
+ .kv span{color:var(--fg-3)} .kv b{font-weight:600;font-variant-numeric:tabular-nums}
238
+
239
+ /* =========================================================================
240
+ Mobile / touch
241
+ ========================================================================= */
242
+ html{-webkit-text-size-adjust:100%}
243
+ body{overflow-x:hidden}
244
+ .tscroll{overflow-x:auto;-webkit-overflow-scrolling:touch}
245
+ .logs{-webkit-overflow-scrolling:touch}
246
+ img,svg{max-width:100%}
247
+
248
+ /* Stat tiles stay two-up rather than becoming a tall stack. */
249
+ @media(max-width:880px){
250
+ .g4{grid-template-columns:repeat(2,1fr)}
251
+ .g3{grid-template-columns:repeat(2,1fr)}
252
+ .stats{grid-template-columns:repeat(2,1fr)}
253
+ }
254
+
255
+ @media(max-width:760px){
256
+ .wrap{padding:0 16px}
257
+ main{padding:26px 0 56px}
258
+
259
+ /* Two-row nav: brand + account on top, scrollable links underneath. */
260
+ nav .wrap{height:auto;flex-wrap:wrap;padding-top:10px;padding-bottom:8px;
261
+ row-gap:6px}
262
+ .brand{margin-right:auto;font-size:16px}
263
+ .navlinks{order:3;flex-basis:100%;overflow-x:auto;
264
+ -webkit-overflow-scrolling:touch;scrollbar-width:none;gap:3px;
265
+ padding-bottom:2px}
266
+ .navlinks::-webkit-scrollbar{display:none}
267
+ .navlinks a{padding:7px 12px;font-size:13.5px;flex:none;white-space:nowrap}
268
+ /* "Sign in with Hugging Face" is too wide for a phone header. */
269
+ .signin-long{display:none}
270
+ nav .btn.primary{padding:9px 14px;min-height:36px}
271
+
272
+ .hero{padding:44px 0 38px}
273
+ .hero .cta{flex-direction:column;align-items:stretch}
274
+ .hero .cta .btn{width:100%}
275
+
276
+ .btn{padding:11px 17px;min-height:42px}
277
+ .btn.sm{padding:8px 13px;min-height:36px}
278
+ .btn.lg{padding:14px 22px;min-height:48px}
279
+
280
+ .card{padding:17px} .card.pad-lg{padding:19px}
281
+ .stat{padding:14px 15px}
282
+ .stat b{font-size:21px}
283
+
284
+ .sect-h{flex-direction:column;align-items:flex-start;gap:8px}
285
+ .sect-h h1,.sect-h h2{font-size:18px}
286
+
287
+ /* Wizard: four steps still fit by shrinking the labels. */
288
+ .steps{margin-bottom:22px}
289
+ .stp{padding-bottom:11px}
290
+ .stp .n{font-size:9.5px;letter-spacing:.06em}
291
+ .stp .t{font-size:12px}
292
+
293
+ .chart{height:210px}
294
+ .run{padding:14px 15px}
295
+ .run-meta{gap:10px;font-size:12px}
296
+ .kv{font-size:13px;gap:10px}
297
+ input[type=text],input[type=number],textarea,select{font-size:16px} /* no iOS zoom */
298
+ input[type=range]{height:8px}
299
+ input[type=range]::-webkit-slider-thumb{width:24px;height:24px}
300
+ input[type=range]::-moz-range-thumb{width:24px;height:24px}
301
+ .logs{font-size:11.6px;max-height:260px}
302
+ th,td{padding-left:10px;padding-right:10px}
303
+ .toast{left:16px;right:16px;transform:translateY(80px);max-width:none}
304
+ .toast.on{transform:none}
305
+ }
306
+
307
+ @media(max-width:520px){
308
+ .picks{grid-template-columns:1fr}
309
+ .g3{grid-template-columns:1fr}
310
+ .pick{display:flex;align-items:baseline;gap:10px;flex-wrap:wrap;padding:14px}
311
+ .pick b{flex:none}
312
+ .pick .p-n{margin:0}
313
+ .pick .p-d{flex-basis:100%;margin-top:4px}
314
+ .hero h1{letter-spacing:-.035em}
315
+ footer .wrap{flex-direction:column;gap:8px;font-size:12.5px}
316
+ }
317
+
318
+ @media(max-width:359px){
319
+ .g4,.stats{grid-template-columns:1fr}
320
+ .stat b{font-size:24px}
321
+ }
322
+
323
+ @media(hover:none){
324
+ .run:active{background:var(--panel-2)}
325
+ .pick:active{border-color:var(--accent)}
326
+ }
327
+
328
+ /* Long labels that collapse on narrow screens */
329
+ .signin-short{display:none}
330
+ @media(max-width:760px){ .signin-short{display:inline} }
331
+
332
+ /* Empty-state text inside the loss chart needs to survive the viewBox scale */
333
+ .chart .ph{fill:var(--fg-3);font-size:15px;font-family:var(--sans)}
334
+
335
+ /* Playground output — prompt dimmed, continuation bright */
336
+ .pgout{background:#05080b;border:1px solid var(--line);border-radius:var(--r-sm);
337
+ padding:14px 16px;font-family:var(--mono);font-size:13px;line-height:1.8;
338
+ white-space:pre-wrap;word-break:break-word;max-height:340px;overflow:auto;
339
+ -webkit-overflow-scrolling:touch}
340
+ .pgout .pgp{color:var(--fg-3)}
341
+ .pgout .pgc{color:#bfe9d8}
342
+ @media(max-width:760px){ .pgout{font-size:12.4px;padding:12px 13px} }
343
+
344
+ /* Shown only when the app is running inside the Space iframe */
345
+ .frame-note{display:none}
346
+ .framed .frame-note{display:block;position:relative;margin:16px auto 0;
347
+ max-width:460px;font-size:12.5px;color:var(--fg-3);text-align:center}
348
+
349
+ /* Token-budget shortcuts under the slider */
350
+ .quickpick{display:flex;gap:7px;margin-top:14px;flex-wrap:wrap}
351
+ .quickpick .btn{flex:1;min-width:64px}
web/static/app.js ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /* NanoDex — shared frontend helpers */
2
+
3
+ const NX = {
4
+ fmtTokens(n){ n = Number(n||0);
5
+ if(n>=1e9) return (n/1e9).toFixed(2)+'B';
6
+ if(n>=1e6) return (n/1e6).toFixed(n>=1e7?0:1).replace(/\.0$/,'')+'M';
7
+ if(n>=1e3) return Math.round(n/1e3)+'k';
8
+ return String(n); },
9
+ fmtParams(n){ n = Number(n||0);
10
+ return n>=1e6 ? (n/1e6).toFixed(2)+'M' : Math.round(n/1e3)+'k'; },
11
+ fmtTime(s){ s = Number(s||0); if(!isFinite(s)||s<0) return '—';
12
+ if(s<90) return Math.round(s)+'s';
13
+ if(s<5400) return Math.round(s/60)+' min';
14
+ return (s/3600).toFixed(1)+' h'; },
15
+ ago(ts){ if(!ts) return '—';
16
+ const d = Date.now()/1000 - ts;
17
+ if(d<60) return 'just now';
18
+ if(d<3600) return Math.floor(d/60)+'m ago';
19
+ if(d<86400) return Math.floor(d/3600)+'h ago';
20
+ return Math.floor(d/86400)+'d ago'; },
21
+ esc(s){ const d=document.createElement('div'); d.textContent=s==null?'':String(s);
22
+ return d.innerHTML; },
23
+
24
+ chip(status, extra){
25
+ const label = {queued:'queued', running:'training', done:'completed',
26
+ failed:'failed', cancelled:'cancelled'}[status] || status;
27
+ return `<span class="chip ${status}"><i></i>${label}${extra?' · '+NX.esc(extra):''}</span>`;
28
+ },
29
+
30
+ async get(url){
31
+ const r = await fetch(url, {headers:{'Accept':'application/json'}});
32
+ if(!r.ok) throw new Error((await r.json().catch(()=>({}))).error || r.statusText);
33
+ return r.json();
34
+ },
35
+ async post(url, body){
36
+ const r = await fetch(url, {method:'POST',
37
+ headers:{'Content-Type':'application/json'}, body: JSON.stringify(body||{})});
38
+ const j = await r.json().catch(()=>({}));
39
+ if(!r.ok) throw new Error(j.error || r.statusText);
40
+ return j;
41
+ },
42
+
43
+ toast(msg, kind){
44
+ let t = document.querySelector('.toast');
45
+ if(!t){ t = document.createElement('div'); t.className='toast';
46
+ document.body.appendChild(t); }
47
+ t.className = 'toast ' + (kind||'');
48
+ t.innerHTML = msg;
49
+ requestAnimationFrame(()=> t.classList.add('on'));
50
+ clearTimeout(t._h);
51
+ t._h = setTimeout(()=> t.classList.remove('on'), 5200);
52
+ },
53
+
54
+ runRow(j){
55
+ const pct = (j.progress*100).toFixed(1);
56
+ const bits = [];
57
+ if(j.status==='running'){
58
+ bits.push(`<span><b>${pct}%</b> of ${NX.fmtTokens(j.target_tokens)}</span>`);
59
+ if(j.loss) bits.push(`<span>loss <b>${j.loss.toFixed(4)}</b></span>`);
60
+ bits.push(`<span><b>${Math.round(j.tok_per_s/1e3)}k</b> tok/s</span>`);
61
+ if(j.eta_s) bits.push(`<span><b>${NX.fmtTime(j.eta_s)}</b> left</span>`);
62
+ } else if(j.status==='queued'){
63
+ bits.push(`<span>position <b>#${j.queue_position||'?'}</b></span>`);
64
+ bits.push(`<span><b>${NX.fmtTokens(j.target_tokens)}</b> tokens queued</span>`);
65
+ } else if(j.status==='done'){
66
+ bits.push(`<span><b>${NX.fmtTokens(j.tokens_seen)}</b> tokens</span>`);
67
+ if(j.loss) bits.push(`<span>final loss <b>${j.loss.toFixed(4)}</b></span>`);
68
+ bits.push(`<span>${j.pushed_repo ? 'published' : 'not published'}</span>`);
69
+ } else {
70
+ bits.push(`<span>${NX.esc(j.error||'—')}</span>`);
71
+ }
72
+ const showBar = j.status==='running';
73
+ return `<a class="run" href="/models/${j.id}">
74
+ <div class="run-top">
75
+ <h4>${NX.esc(j.name)}</h4>
76
+ ${NX.chip(j.status)}
77
+ </div>
78
+ <div class="run-meta">
79
+ <span class="mono">${NX.esc(j.tier_label)}</span>
80
+ <span>${Number(j.n_params).toLocaleString()} params</span>
81
+ ${bits.join('')}
82
+ <span class="muted">${NX.ago(j.created_at)}</span>
83
+ </div>
84
+ ${showBar ? `<div class="bar"><i style="width:${pct}%"></i></div>` : ''}
85
+ </a>`;
86
+ },
87
+
88
+ /* Minimal dependency-free loss chart.
89
+ The viewBox is set to the element's real pixel size so that 1 user unit ==
90
+ 1 CSS pixel: a fixed 760-wide viewBox squeezed into a 280px phone would
91
+ render the axis labels at ~4px. */
92
+ lossChart(el, pts){
93
+ if(!el) return;
94
+ el._pts = pts;
95
+ const rect = el.getBoundingClientRect();
96
+ const W = Math.max(280, Math.round(rect.width) || 760);
97
+ const H = Math.max(170, Math.round(rect.height) || 280);
98
+ const narrow = W < 460;
99
+ const PL = narrow ? 40 : 54, PR = narrow ? 8 : 14;
100
+ const PT = 16, PB = narrow ? 28 : 34;
101
+ const TICKS = narrow ? 3 : 4;
102
+ el.setAttribute('viewBox', `0 0 ${W} ${H}`);
103
+ if(!pts || pts.length < 2){
104
+ el.innerHTML = `<text x="${W/2}" y="${H/2}" class="ph"
105
+ text-anchor="middle">waiting for the first metrics…</text>`;
106
+ return;
107
+ }
108
+ const xs = pts.map(p=>p.tokens), ys = pts.map(p=>p.loss);
109
+ const x0=Math.min(...xs), x1=Math.max(...xs);
110
+ let y0=Math.min(...ys), y1=Math.max(...ys);
111
+ const padY=(y1-y0)*0.12 || 0.4; y0-=padY; y1+=padY;
112
+ const sx = v => PL + (v-x0)/((x1-x0)||1)*(W-PL-PR);
113
+ const sy = v => PT + (1-(v-y0)/((y1-y0)||1))*(H-PT-PB);
114
+ const d = pts.map((p,i)=> (i?'L':'M')+sx(p.tokens).toFixed(1)+' '+sy(p.loss).toFixed(1)).join(' ');
115
+ const area = d + ` L${sx(x1).toFixed(1)} ${H-PB} L${sx(x0).toFixed(1)} ${H-PB} Z`;
116
+ let g='';
117
+ for(let i=0;i<=TICKS;i++){
118
+ const v = y0+(y1-y0)*i/TICKS, y = sy(v);
119
+ g += `<line class="gl" x1="${PL}" y1="${y.toFixed(1)}" x2="${W-PR}" y2="${y.toFixed(1)}"/>`;
120
+ g += `<text class="ax" x="${PL-7}" y="${(y+3.5).toFixed(1)}" text-anchor="end">${v.toFixed(2)}</text>`;
121
+ }
122
+ for(let i=0;i<=TICKS;i++){
123
+ const v = x0+(x1-x0)*i/TICKS, x = sx(v);
124
+ const anchor = i===0 ? 'start' : (i===TICKS ? 'end' : 'middle');
125
+ g += `<text class="ax" x="${x.toFixed(1)}" y="${H-PB+18}" text-anchor="${anchor}">${NX.fmtTokens(v)}</text>`;
126
+ }
127
+ const last = pts[pts.length-1];
128
+ el.innerHTML = `<defs><linearGradient id="lg" x1="0" y1="0" x2="0" y2="1">
129
+ <stop offset="0%" stop-color="#2ee6a8" stop-opacity=".22"/>
130
+ <stop offset="100%" stop-color="#2ee6a8" stop-opacity="0"/>
131
+ </linearGradient></defs>
132
+ ${g}
133
+ <path class="ar" d="${area}"/>
134
+ <path class="ln" d="${d}"/>
135
+ <circle cx="${sx(last.tokens).toFixed(1)}" cy="${sy(last.loss).toFixed(1)}"
136
+ r="4" fill="#2ee6a8"/>
137
+ <text class="ax" x="${W-PR}" y="${PT+2}" text-anchor="end"
138
+ style="fill:#2ee6a8;font-size:12px">loss ${last.loss.toFixed(4)}</text>`;
139
+ },
140
+
141
+ poll(fn, ms){
142
+ let stop=false;
143
+ const tick = async () => {
144
+ if(stop) return;
145
+ try { await fn(); } catch(e){ /* keep polling */ }
146
+ if(!stop) setTimeout(tick, ms);
147
+ };
148
+ tick();
149
+ return ()=>{ stop=true; };
150
+ }
151
+ };
152
+ window.NX = NX;
153
+
154
+ /* Charts use pixel-sized viewBoxes, so they must be redrawn when the box
155
+ changes — rotating a phone, or opening the keyboard. */
156
+ let _rsT;
157
+ addEventListener('resize', () => {
158
+ clearTimeout(_rsT);
159
+ _rsT = setTimeout(() => {
160
+ document.querySelectorAll('svg.chart').forEach(el => {
161
+ if(el._pts) NX.lossChart(el, el._pts);
162
+ });
163
+ }, 160);
164
+ });
web/templates/about.html ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'about' %}
3
+ {% block title %}How it works — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div style="max-width:760px">
7
+ <span class="eyebrow">How it works</span>
8
+ <h1 style="font-size:30px;margin:6px 0 10px">What actually happens when you
9
+ press the button</h1>
10
+ <p class="sub" style="font-size:16px">No fine-tuning, no adapters, no
11
+ distillation. A transformer is created with random weights and taught to
12
+ predict the next token, from zero.</p>
13
+ </div>
14
+
15
+ <section style="margin-top:36px">
16
+ <div class="card pad-lg">
17
+ <div class="step-n">1</div>
18
+ <h2 style="font-size:19px">Architecture</h2>
19
+ <p class="sub" style="margin-top:8px">A standard modern decoder-only
20
+ transformer — <span class="mono">LlamaForCausalLM</span>: SiLU MLPs, RMSNorm
21
+ (ε=1e-5), rotary position embeddings (θ=10000), grouped-query attention,
22
+ tied input/output embeddings, no biases. The four tiers are that same
23
+ recipe, narrowed and shortened until it fits the parameter budget.</p>
24
+
25
+ <div style="overflow-x:auto;margin-top:18px">
26
+ <table>
27
+ <thead><tr><th>Tier</th><th>Parameters</th><th>Layers</th><th>Hidden</th>
28
+ <th>Heads (KV)</th><th>FFN</th><th>Tokens / step</th></tr></thead>
29
+ <tbody>
30
+ {% for t in tiers %}
31
+ <tr><td><b>{{ t.label }}</b></td>
32
+ <td class="mono">{{ "{:,}".format(t.params) }}</td>
33
+ <td>{{ t.layers }}</td><td>{{ t.hidden }}</td>
34
+ <td>{{ t.heads }} ({{ t.kv_heads }})</td><td>{{ t.ffn }}</td>
35
+ <td class="mono">{{ "{:,}".format(t.batch_tokens) }}</td></tr>
36
+ {% endfor %}
37
+ </tbody>
38
+ </table>
39
+ </div>
40
+ </div>
41
+ </section>
42
+
43
+ <section style="margin-top:16px">
44
+ <div class="card pad-lg">
45
+ <div class="step-n">2</div>
46
+ <h2 style="font-size:19px">Tokenizer</h2>
47
+ <p class="sub" style="margin-top:8px">A byte-level BPE with
48
+ <b>{{ "{:,}".format(vocab) }} tokens</b>, trained on fineweb-edu itself
49
+ (~2.7 characters per token). This matters more than it sounds: a typical
50
+ small-LM vocabulary of ~49k tokens would cost 28M embedding parameters on
51
+ its own — more than three times the largest model here. A lean vocabulary is
52
+ what makes "a 500k-parameter language model" a sentence that means something. The
53
+ parameter counts above are real totals, embeddings included.</p>
54
+ <p class="sub" style="margin-top:12px">The vocabulary is deliberately the
55
+ same for every tier, and stays at 2,048 even for the 8M model. Two reasons:
56
+ at a 500k total budget anything larger would be almost entirely
57
+ embeddings, and a shared vocabulary is the only thing that makes the
58
+ losses on the <a href="/explore" style="color:var(--accent)">leaderboard</a>
59
+ comparable at all — change the tokenizer and cross-entropy stops meaning
60
+ the same thing between two models.</p>
61
+ </div>
62
+ </section>
63
+
64
+ <section style="margin-top:16px">
65
+ <div class="card pad-lg">
66
+ <div class="step-n">3</div>
67
+ <h2 style="font-size:19px">Data</h2>
68
+ <p class="sub" style="margin-top:8px">
69
+ <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu"
70
+ target="_blank" rel="noopener" style="color:var(--accent)">HuggingFaceFW/fineweb-edu</a>,
71
+ config <span class="mono">sample-10BT</span> — filtered educational web text.
72
+ Instead of every run re-streaming and re-tokenizing the corpus, this site
73
+ tokenizes a {{ "%.2fB"|format(cache_target / 1000000000) }}-token slice
74
+ <b>once</b> into a flat <span class="mono">uint16</span> file
75
+ (~{{ "%.1f"|format(cache_target * 2 / 1024**3) }} GB) that all runs read
76
+ from, so the workers are never waiting on data. Building it is a one-time
77
+ cost per container; a run that asks for more tokens than are cached yet
78
+ waits for the writer to catch up, and says so in its log. Every run starts at
79
+ its own random offset, so two identical configurations don't see identical
80
+ token order.</p>
81
+ </div>
82
+ </section>
83
+
84
+ <section style="margin-top:16px">
85
+ <div class="card pad-lg">
86
+ <div class="step-n">4</div>
87
+ <h2 style="font-size:19px">Training</h2>
88
+ <div class="grid g2" style="margin-top:14px">
89
+ <div>
90
+ <div class="kv"><span>Optimizer</span><b>AdamW β=(0.9, 0.95)</b></div>
91
+ <div class="kv"><span>Weight decay</span><b>0.1</b></div>
92
+ <div class="kv"><span>Gradient clipping</span><b>1.0</b></div>
93
+ <div class="kv"><span>Schedule</span><b>2% warmup → cosine to 10%</b></div>
94
+ </div>
95
+ <div>
96
+ <div class="kv"><span>Precision</span><b>bf16 autocast on GPU</b></div>
97
+ <div class="kv"><span>Context length</span><b>{{ seq_len }} tokens</b></div>
98
+ <div class="kv"><span>Token budget</span>
99
+ <b>{{ min_tokens // 10**6 }}M – 1.5B</b></div>
100
+ <div class="kv"><span>Concurrent runs</span>
101
+ <b>{{ max_active }} per person</b></div>
102
+ </div>
103
+ </div>
104
+ </div>
105
+ </section>
106
+
107
+ <section style="margin-top:16px">
108
+ <div class="card pad-lg">
109
+ <div class="step-n">5</div>
110
+ <h2 style="font-size:19px">Queue</h2>
111
+ <p class="sub" style="margin-top:8px">One worker thread per available device
112
+ claims the oldest queued run atomically, so two workers can never grab the
113
+ same job. This deployment currently has
114
+ <b>{{ n_devices }} {{ 'GPU' if on_gpu else 'CPU' }} worker{{ 's' if n_devices != 1 }}</b>,
115
+ which means {{ n_devices }} run{{ 's' if n_devices != 1 }} can train at
116
+ once. Estimated times shown in the wizard start from a prior and then
117
+ recalibrate from the median throughput of runs that actually finished here.</p>
118
+ </div>
119
+ </section>
120
+
121
+ <section style="margin-top:16px">
122
+ <div class="card pad-lg">
123
+ <div class="step-n">6</div>
124
+ <h2 style="font-size:19px">Publishing</h2>
125
+ <p class="sub" style="margin-top:8px">When a run finishes you can push it to
126
+ your own Hugging Face namespace with the OAuth token from your session:
127
+ <span class="mono">config.json</span>,
128
+ <span class="mono">model.safetensors</span>, tokenizer files,
129
+ <span class="mono">generation_config.json</span>, a
130
+ <span class="mono">training_run.json</span> recording the complete recipe,
131
+ and a generated model card. It loads with
132
+ <span class="mono">transformers</span> like any other model on the Hub.</p>
133
+ </div>
134
+ </section>
135
+
136
+ <section style="margin-top:16px">
137
+ <div class="card pad-lg">
138
+ <div class="step-n">7</div>
139
+ <h2 style="font-size:19px">Durable storage</h2>
140
+ <p class="sub" style="margin-top:8px">A Space's filesystem is wiped on every
141
+ rebuild and restart, so finished runs, their loss curves and the token
142
+ cache are mirrored to a Hugging Face bucket. The archive is append-only
143
+ and keyed by run id — a finished run is written once and never mutated,
144
+ which is why a rolling deploy running two containers at once cannot
145
+ corrupt it. History comes back on boot; weights are pulled on demand the
146
+ first time somebody opens the playground.</p>
147
+ <p class="sub" style="margin-top:12px">Runs that are mid-training when a
148
+ container dies are <b>not</b> recoverable — the optimizer state only ever
149
+ exists in that process. They come back marked interrupted.</p>
150
+ </div>
151
+ </section>
152
+
153
+ <section style="margin-top:36px">
154
+ <div class="card pad-lg" style="border-color:var(--line-2)">
155
+ <span class="eyebrow">A note on scale</span>
156
+ <h2 style="font-size:20px;margin:8px 0 10px">Deliberately over-trained</h2>
157
+ <p class="sub">1.5 billion tokens into an 8M-parameter model is roughly 9×
158
+ past the Chinchilla compute-optimal ratio of 20 tokens per parameter —
159
+ and into a 500k-parameter model, over 150× past it. That's intentional.
160
+ Tiny models keep improving long after the
161
+ compute-optimal point, and the goal here was never FLOP efficiency — it's
162
+ watching cross-entropy fall from <b>7.6</b> (uniform noise over
163
+ {{ "{:,}".format(vocab) }} tokens) toward <b>4</b>, and seeing a network
164
+ that started as pure randomness begin to emit English words.</p>
165
+ <p class="sub" style="margin-top:12px">Nothing trained here is a useful
166
+ assistant, and nothing it says is factual. That was never the point.</p>
167
+ </div>
168
+ </section>
169
+
170
+ {% endblock %}
web/templates/base.html ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>{% block title %}NanoDex{% endblock %}</title>
7
+ <meta name="description" content="Pre-train a small decoder-only language model from scratch, in the cloud.">
8
+ <link rel="stylesheet" href="/static/app.css">
9
+ <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>&#129516;</text></svg>">
10
+ {% block head %}{% endblock %}
11
+ </head>
12
+ <body>
13
+
14
+ <nav>
15
+ <div class="wrap">
16
+ <a class="brand" href="/"><span class="dot"></span>NanoDex</a>
17
+ <div class="navlinks">
18
+ {% if user %}
19
+ <a href="/" class="{{ 'on' if nav=='home' }}">Home</a>
20
+ <a href="/models" class="{{ 'on' if nav=='models' }}">My models</a>
21
+ <a href="/train" class="{{ 'on' if nav=='train' }}">New run</a>
22
+ {% endif %}
23
+ <a href="/explore" class="{{ 'on' if nav=='explore' }}">Explore</a>
24
+ <a href="/queue" class="{{ 'on' if nav=='queue' }}">Queue</a>
25
+ <a href="/about" class="{{ 'on' if nav=='about' }}">How it works</a>
26
+ </div>
27
+ {% if user %}
28
+ <a class="usermenu" href="/profile">
29
+ {% if user.picture %}<img class="avatar" src="{{ user.picture }}" alt="">
30
+ {% else %}<span class="avatar"></span>{% endif %}
31
+ <span>{{ user.username }}</span>
32
+ </a>
33
+ {% else %}
34
+ <a class="btn primary sm" target="_top" href="/login"><span class="signin-long">Sign in with Hugging Face</span><span class="signin-short">Sign in</span></a>
35
+ {% endif %}
36
+ </div>
37
+ </nav>
38
+
39
+ <main><div class="wrap">{% block body %}{% endblock %}</div></main>
40
+
41
+ <footer><div class="wrap">
42
+ <div>NanoDex · train a decoder-only language model from scratch</div>
43
+ <div>
44
+ <a href="/about">How it works</a> ·
45
+ <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu" target="_blank" rel="noopener">fineweb-edu</a> ·
46
+ <a href="https://huggingface.co/hugging-science" target="_blank" rel="noopener">hugging-science</a>
47
+ </div>
48
+ </div></footer>
49
+
50
+ <script>
51
+ /* huggingface.co refuses to be displayed in a frame, so the OAuth handshake
52
+ must run as a top-level navigation — inside the Space iframe it would just
53
+ render a blank grey box. Retarget anything we missed, and show a hint. */
54
+ (function(){
55
+ if (window.self === window.top) return;
56
+ document.documentElement.classList.add('framed');
57
+ addEventListener('DOMContentLoaded', function(){
58
+ document.querySelectorAll('a[href^="/login"], a[href^="/logout"]')
59
+ .forEach(function(a){ a.target = '_top'; });
60
+ });
61
+ })();
62
+ </script>
63
+ <script src="/static/app.js"></script>
64
+ {% block scripts %}{% endblock %}
65
+ </body>
66
+ </html>
web/templates/detail.html ADDED
@@ -0,0 +1,319 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'models' %}
3
+ {% block title %}{{ job.name }} — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <a href="{{ '/models' if owner else '/explore' }}" class="btn ghost sm"
7
+ style="margin-bottom:16px">← {{ 'My models' if owner else 'Explore' }}</a>
8
+
9
+ <div class="sect-h" style="align-items:flex-start;margin-bottom:20px">
10
+ <div>
11
+ <div style="display:flex;align-items:center;gap:11px;flex-wrap:wrap">
12
+ <h1 style="font-size:27px">{{ job.name }}</h1>
13
+ <span id="statusChip"></span>
14
+ </div>
15
+ <p class="sub" style="margin-top:5px">
16
+ {{ job.tier_label }} · {{ "{:,}".format(job.n_params) }} parameters ·
17
+ by <a href="/u/{{ job.username }}" style="color:var(--accent)">@{{ job.username }}</a>
18
+ </p>
19
+ </div>
20
+ {% if owner %}
21
+ <div style="display:flex;gap:8px">
22
+ <button class="btn danger sm" id="cancelBtn" style="display:none">Cancel run</button>
23
+ </div>
24
+ {% endif %}
25
+ </div>
26
+
27
+ <div id="banner"></div>
28
+
29
+ <div class="card" style="margin-bottom:16px">
30
+ <div style="display:flex;justify-content:space-between;align-items:baseline;margin-bottom:10px">
31
+ <span class="eyebrow">Progress</span>
32
+ <span class="mono" id="pctLabel" style="color:var(--accent);font-weight:640">—</span>
33
+ </div>
34
+ <div class="bar"><i id="progBar" style="width:0%"></i></div>
35
+ <div class="grid g4" style="margin-top:18px;gap:14px">
36
+ <div><span class="eyebrow">Tokens seen</span>
37
+ <div class="mono" style="font-size:20px;font-weight:650" id="m-tokens">—</div>
38
+ <small class="muted" id="m-target">&nbsp;</small></div>
39
+ <div><span class="eyebrow">Step</span>
40
+ <div class="mono" style="font-size:20px;font-weight:650" id="m-step">—</div>
41
+ <small class="muted" id="m-steps">&nbsp;</small></div>
42
+ <div><span class="eyebrow">Loss</span>
43
+ <div class="mono" style="font-size:20px;font-weight:650" id="m-loss">—</div>
44
+ <small class="muted" id="m-ppl">&nbsp;</small></div>
45
+ <div><span class="eyebrow">Throughput</span>
46
+ <div class="mono" style="font-size:20px;font-weight:650" id="m-tps">—</div>
47
+ <small class="muted" id="m-eta">&nbsp;</small></div>
48
+ </div>
49
+ </div>
50
+
51
+ <div class="card" style="margin-bottom:16px">
52
+ <div class="sect-h" style="margin-bottom:8px">
53
+ <h2 style="font-size:16px">Loss curve</h2>
54
+ <span class="sub">cross-entropy vs tokens seen</span>
55
+ </div>
56
+ <svg class="chart" id="chart" preserveAspectRatio="xMidYMid meet"></svg>
57
+ </div>
58
+
59
+ <div class="grid g2" style="margin-bottom:16px">
60
+ <div class="card">
61
+ <h2 style="font-size:16px;margin-bottom:12px">Architecture</h2>
62
+ <div class="kv"><span>Class</span><b class="mono">LlamaForCausalLM</b></div>
63
+ <div class="kv"><span>Layers</span><b>{{ job.arch.layers }}</b></div>
64
+ <div class="kv"><span>Hidden size</span><b>{{ job.arch.hidden }}</b></div>
65
+ <div class="kv"><span>Attention heads</span><b>{{ job.arch.heads }} ({{ job.arch.kv_heads }} KV, head dim {{ job.arch.head_dim }})</b></div>
66
+ <div class="kv"><span>FFN size</span><b>{{ job.arch.ffn }}</b></div>
67
+ <div class="kv"><span>Context</span><b>{{ job.arch.ctx }} tokens</b></div>
68
+ <div class="kv"><span>Vocabulary</span><b>{{ "{:,}".format(job.arch.vocab) }}</b></div>
69
+ <div class="kv"><span>Parameters</span><b>{{ "{:,}".format(job.n_params) }}</b></div>
70
+ </div>
71
+ <div class="card">
72
+ <h2 style="font-size:16px;margin-bottom:12px">Run</h2>
73
+ <div class="kv"><span>Dataset</span><b>fineweb-edu</b></div>
74
+ <div class="kv"><span>Token budget</span><b id="d-target">—</b></div>
75
+ <div class="kv"><span>Optimizer</span><b>AdamW β=0.9/0.95 wd=0.1</b></div>
76
+ <div class="kv"><span>Schedule</span><b>2% warmup → cosine</b></div>
77
+ <div class="kv"><span>Learning rate</span><b id="d-lr">—</b></div>
78
+ <div class="kv"><span>Best loss</span><b id="d-best">—</b></div>
79
+ <div class="kv"><span>Worker</span><b id="d-gpu">—</b></div>
80
+ <div class="kv"><span>Queued</span><b id="d-created">—</b></div>
81
+ </div>
82
+ </div>
83
+
84
+ {% if owner %}
85
+ <div class="card" id="publishCard" style="display:none;margin-bottom:16px">
86
+ <h2 style="font-size:16px">Publish to your Hugging Face account</h2>
87
+ <p class="sub" style="margin:6px 0 14px">Uploads weights, config, tokenizer, the
88
+ full training recipe and a generated model card to
89
+ <span class="mono">{{ user.username }}/&lt;name&gt;</span>.</p>
90
+ <div style="display:flex;gap:10px;flex-wrap:wrap">
91
+ <input type="text" id="repoName" placeholder="{{ job.name }}" style="flex:1;min-width:220px">
92
+ <button class="btn primary" id="publishBtn">Publish</button>
93
+ </div>
94
+ </div>
95
+ {% endif %}
96
+
97
+ <div class="card" id="pgCard" style="display:none">
98
+ <div class="sect-h" style="margin-bottom:6px">
99
+ <h2 style="font-size:16px">Playground</h2>
100
+ <span id="ckptChip"></span>
101
+ </div>
102
+ <p class="sub" style="margin:0 0 14px" id="pgBlurb">Generate text with this
103
+ model. At this scale expect English-shaped output with no facts in it —
104
+ that's the honest result.</p>
105
+ <label class="fl">Prompt</label>
106
+ <textarea id="pgPrompt" rows="2">The mitochondria is</textarea>
107
+ <div class="grid g4" style="margin-top:14px">
108
+ <div><label class="fl">New tokens <span class="mono" id="v-max">80</span></label>
109
+ <input type="range" id="pgMax" min="8" max="256" step="8" value="80"></div>
110
+ <div><label class="fl">Temperature <span class="mono" id="v-temp">0.85</span></label>
111
+ <input type="range" id="pgTemp" min="0.1" max="2" step="0.05" value="0.85"></div>
112
+ <div><label class="fl">Top-k <span class="mono" id="v-topk">40</span></label>
113
+ <input type="range" id="pgTopk" min="0" max="200" step="5" value="40"></div>
114
+ <div><label class="fl">Top-p <span class="mono" id="v-topp">0.95</span></label>
115
+ <input type="range" id="pgTopp" min="0.1" max="1" step="0.05" value="0.95"></div>
116
+ </div>
117
+ <div style="display:flex;gap:10px;align-items:center;margin-top:16px;flex-wrap:wrap">
118
+ {% if user %}<button class="btn primary" id="pgGo">Generate</button>
119
+ {% else %}<a class="btn primary" target="_top"
120
+ href="/login?next=/models/{{ job.id }}">Sign in to generate</a>{% endif %}
121
+ {% if user %}
122
+ <label class="sub" style="display:flex;align-items:center;gap:7px;font-size:13px">
123
+ <input type="checkbox" id="pgAuto" style="width:auto;accent-color:var(--accent)">
124
+ re-run on every new checkpoint
125
+ </label>
126
+ {% endif %}
127
+ <span class="mono muted" id="pgStats" style="font-size:12px"></span>
128
+ </div>
129
+ <div id="pgOut" class="pgout" style="margin-top:14px;display:none"></div>
130
+ </div>
131
+
132
+ <div class="card" style="margin-top:16px">
133
+ <h2 style="font-size:16px;margin-bottom:12px">Logs</h2>
134
+ <div class="logs" id="logs">waiting…</div>
135
+ </div>
136
+
137
+ {% endblock %}
138
+ {% block scripts %}
139
+ <script>
140
+ const JOB_ID = {{ job.id | tojson }};
141
+ const OWNER = {{ owner | tojson }};
142
+ const SIGNED_IN = {{ (user is not none) | tojson }};
143
+ const INFER_DEVICE = {{ infer_device | tojson }};
144
+ let stopPoll = null;
145
+ let lastCkpt = null;
146
+
147
+ function banner(j){
148
+ const el = document.getElementById('banner');
149
+ if(j.status === 'queued'){
150
+ el.innerHTML = `<div class="card" style="margin-bottom:16px;border-color:#3b2f63">
151
+ <b>Position #${j.queue_position || '?'} in the queue.</b>
152
+ <span class="sub"> A worker will pick it up automatically — you can close this tab.</span>
153
+ </div>`;
154
+ } else if(j.status === 'failed'){
155
+ el.innerHTML = `<div class="card" style="margin-bottom:16px;border-color:#4c1d24">
156
+ <b style="color:var(--red)">This run failed.</b>
157
+ <div class="mono sub" style="margin-top:6px">${NX.esc(j.error||'')}</div></div>`;
158
+ } else if(j.status === 'done' && j.pushed_repo){
159
+ el.innerHTML = `<div class="card" style="margin-bottom:16px;border-color:var(--accent-2)">
160
+ <b>Published →</b> <a class="mono" style="color:var(--accent)"
161
+ href="https://huggingface.co/${NX.esc(j.pushed_repo)}" target="_blank"
162
+ rel="noopener">${NX.esc(j.pushed_repo)}</a></div>`;
163
+ } else if(j.status === 'cancelled'){
164
+ el.innerHTML = `<div class="card" style="margin-bottom:16px">
165
+ <b>Cancelled.</b> <span class="sub">Stopped before finishing.</span></div>`;
166
+ } else { el.innerHTML = ''; }
167
+ }
168
+
169
+ async function refresh(){
170
+ const j = await NX.get('/api/jobs/' + JOB_ID);
171
+ const pct = (j.progress * 100);
172
+
173
+ document.getElementById('statusChip').innerHTML = NX.chip(j.status);
174
+ document.getElementById('pctLabel').textContent = pct.toFixed(1) + '%';
175
+ document.getElementById('progBar').style.width = pct.toFixed(2) + '%';
176
+ document.getElementById('m-tokens').textContent = NX.fmtTokens(j.tokens_seen);
177
+ document.getElementById('m-target').textContent = 'of ' + NX.fmtTokens(j.target_tokens);
178
+ document.getElementById('m-step').textContent = Number(j.step).toLocaleString();
179
+ document.getElementById('m-steps').textContent = 'of ' + Number(j.total_steps).toLocaleString();
180
+ document.getElementById('m-loss').textContent = j.loss ? j.loss.toFixed(4) : '—';
181
+ document.getElementById('m-ppl').textContent = j.loss
182
+ ? 'ppl ' + Math.exp(Math.min(j.loss,20)).toFixed(1) : 'not started';
183
+ document.getElementById('m-tps').textContent = j.tok_per_s
184
+ ? Math.round(j.tok_per_s/1e3) + 'k/s' : '—';
185
+ document.getElementById('m-eta').textContent = (j.status==='running' && j.eta_s)
186
+ ? NX.fmtTime(j.eta_s) + ' left' : (j.status==='done' ? 'completed' : '');
187
+
188
+ document.getElementById('d-target').textContent = NX.fmtTokens(j.target_tokens) + ' tokens';
189
+ document.getElementById('d-lr').textContent = j.lr ? j.lr.toExponential(2) : '—';
190
+ document.getElementById('d-best').textContent = j.best_loss ? j.best_loss.toFixed(4) : '—';
191
+ document.getElementById('d-gpu').textContent =
192
+ j.gpu_index === null || j.gpu_index === undefined ? 'not assigned' : ('worker ' + j.gpu_index);
193
+ document.getElementById('d-created').textContent = NX.ago(j.created_at);
194
+
195
+ banner(j);
196
+
197
+ if(OWNER){
198
+ const cb = document.getElementById('cancelBtn');
199
+ if(cb) cb.style.display =
200
+ (j.status==='queued' || j.status==='running') ? 'inline-flex' : 'none';
201
+ const pc = document.getElementById('publishCard');
202
+ if(pc) pc.style.display = (j.status==='done' && !j.pushed_repo) ? 'block' : 'none';
203
+ }
204
+
205
+ {
206
+ // The playground is available to anyone signed in, from the moment the
207
+ // first live checkpoint lands — so you can hear someone else's model
208
+ // mid-training, not only your own at the end.
209
+ const pg = document.getElementById('pgCard');
210
+ if(pg) pg.style.display = j.has_checkpoint ? 'block' : 'none';
211
+
212
+ const chip = document.getElementById('ckptChip');
213
+ const blurb = document.getElementById('pgBlurb');
214
+ if(chip){
215
+ if(j.status === 'done'){
216
+ chip.innerHTML = `<span class="chip done"><i></i>final weights</span>`;
217
+ if(blurb) blurb.textContent = OWNER
218
+ ? "Generate text with the model you trained. At this scale expect "
219
+ + "English-shaped output with no facts in it — that's the honest result."
220
+ : "Generate text with this finished model. At this scale expect "
221
+ + "English-shaped output with no facts in it — that's the honest result.";
222
+ } else if(j.live_step !== null && j.live_step !== undefined){
223
+ chip.innerHTML = `<span class="chip running"><i></i>live checkpoint ·
224
+ step ${Number(j.live_step).toLocaleString()}</span>`;
225
+ if(blurb) blurb.textContent =
226
+ `Weights as of step ${Number(j.live_step).toLocaleString()} of `
227
+ + `${Number(j.total_steps).toLocaleString()}, loaded on ${INFER_DEVICE}. `
228
+ + "It gets less incoherent as the run goes on — generate again later "
229
+ + "and compare.";
230
+ } else {
231
+ chip.innerHTML = '';
232
+ }
233
+ }
234
+
235
+ // auto re-run when a fresher snapshot appears
236
+ const auto = SIGNED_IN ? document.getElementById('pgAuto') : null;
237
+ if(auto && auto.checked && j.live_step !== lastCkpt && j.live_step != null){
238
+ lastCkpt = j.live_step;
239
+ generate(true);
240
+ } else if(j.live_step != null && lastCkpt === null){
241
+ lastCkpt = j.live_step;
242
+ }
243
+ }
244
+
245
+ const {metrics} = await NX.get(`/api/jobs/${JOB_ID}/metrics`);
246
+ NX.lossChart(document.getElementById('chart'), metrics);
247
+
248
+ const {logs} = await NX.get(`/api/jobs/${JOB_ID}/logs`);
249
+ const box = document.getElementById('logs');
250
+ const atBottom = box.scrollHeight - box.scrollTop - box.clientHeight < 40;
251
+ box.innerHTML = logs.length ? logs.map(l =>
252
+ `<span class="t">${new Date(l.t*1000).toLocaleTimeString()}</span>${NX.esc(l.line)}`
253
+ ).join('\n') : 'no logs yet';
254
+ if(atBottom) box.scrollTop = box.scrollHeight;
255
+
256
+ if(['done','failed','cancelled'].includes(j.status) && stopPoll){ stopPoll(); stopPoll = null; }
257
+ }
258
+
259
+ stopPoll = NX.poll(refresh, 3000);
260
+
261
+ const cb = document.getElementById('cancelBtn');
262
+ if(cb) cb.addEventListener('click', async () => {
263
+ if(!confirm('Stop this run? Progress is lost.')) return;
264
+ cb.disabled = true;
265
+ try{ await NX.post(`/api/jobs/${JOB_ID}/cancel`); NX.toast('Cancellation requested.','ok'); }
266
+ catch(e){ NX.toast(NX.esc(e.message),'err'); cb.disabled = false; }
267
+ });
268
+
269
+ const pb = document.getElementById('publishBtn');
270
+ if(pb) pb.addEventListener('click', async () => {
271
+ pb.disabled = true; pb.textContent = 'Uploading…';
272
+ try{
273
+ const r = await NX.post(`/api/jobs/${JOB_ID}/publish`,
274
+ {repo: document.getElementById('repoName').value});
275
+ NX.toast(`Published → <a href="${r.url}" target="_blank"
276
+ style="color:var(--accent)">${NX.esc(r.repo)}</a>`, 'ok');
277
+ if(!stopPoll) refresh();
278
+ }catch(e){ NX.toast(NX.esc(e.message),'err'); }
279
+ finally{ pb.disabled = false; pb.textContent = 'Publish'; }
280
+ });
281
+
282
+ ['Max','Temp','Topk','Topp'].forEach(k => {
283
+ const el = document.getElementById('pg'+k);
284
+ if(!el) return;
285
+ const out = document.getElementById('v-'+k.toLowerCase());
286
+ el.addEventListener('input', () => out.textContent = el.value);
287
+ });
288
+
289
+ const pgGo = document.getElementById('pgGo');
290
+
291
+ async function generate(quiet){
292
+ if(!pgGo || pgGo.disabled) return;
293
+ pgGo.disabled = true;
294
+ const label = pgGo.textContent;
295
+ pgGo.textContent = 'Generating…';
296
+ const out = document.getElementById('pgOut');
297
+ try{
298
+ const r = await NX.post(`/api/jobs/${JOB_ID}/generate`, {
299
+ prompt: document.getElementById('pgPrompt').value,
300
+ max_new_tokens: Number(document.getElementById('pgMax').value),
301
+ temperature: Number(document.getElementById('pgTemp').value),
302
+ top_k: Number(document.getElementById('pgTopk').value),
303
+ top_p: Number(document.getElementById('pgTopp').value),
304
+ });
305
+ out.style.display = 'block';
306
+ // The prompt is dimmed so it reads as the seed, not as model output.
307
+ out.innerHTML = `<span class="pgp">${NX.esc(r.prompt)}</span>`
308
+ + `<span class="pgc">${NX.esc(r.continuation)}</span>`;
309
+ document.getElementById('pgStats').textContent =
310
+ `${r.new_tokens} tokens · ${r.ms} ms · ${r.device} · `
311
+ + (r.checkpoint === 'final' ? 'final weights'
312
+ : `step ${Number(r.step).toLocaleString()}`);
313
+ }catch(e){ if(!quiet) NX.toast(NX.esc(e.message),'err'); }
314
+ finally{ pgGo.disabled = false; pgGo.textContent = label; }
315
+ }
316
+
317
+ if(pgGo) pgGo.addEventListener('click', () => generate(false));
318
+ </script>
319
+ {% endblock %}
web/templates/error.html ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% block title %}{{ code }} — NanoDex{% endblock %}
3
+ {% block body %}
4
+ <div style="padding:80px 0;text-align:center">
5
+ <div class="mono" style="font-weight:700;color:var(--line-2);
6
+ letter-spacing:-.04em;line-height:1.1;
7
+ font-size:{{ '56px' if code|string|length <= 4 else '28px' }}">{{ code }}</div>
8
+ <p class="sub" style="margin:12px 0 26px;font-size:16px">{{ detail }}</p>
9
+ <a class="btn primary" href="/">Back home</a>
10
+ </div>
11
+ {% endblock %}
web/templates/explore.html ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'explore' %}
3
+ {% block title %}Explore — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div class="sect-h" style="margin-bottom:22px">
7
+ <div>
8
+ <span class="eyebrow">The shelf</span>
9
+ <h1 style="font-size:28px;margin-top:5px">Every model trained here</h1>
10
+ <p class="sub" style="margin-top:5px">Same tokenizer, same corpus, same
11
+ context length for everyone — so the losses below compare directly. Open
12
+ any model and generate text from it.</p>
13
+ </div>
14
+ </div>
15
+
16
+ <div class="grid g4" style="margin-bottom:24px">
17
+ <div class="stat"><span>Models</span><b id="s-done">—</b><small>finished runs</small></div>
18
+ <div class="stat"><span>Training now</span><b id="s-run">—</b><small>on workers</small></div>
19
+ <div class="stat"><span>Trainers</span><b id="s-users">—</b><small>people</small></div>
20
+ <div class="stat"><span>Tokens burned</span><b id="s-tok">—</b><small>all time</small></div>
21
+ </div>
22
+
23
+ <section id="liveSec" style="margin-bottom:32px;display:none">
24
+ <div class="sect-h"><h2>Being born right now</h2></div>
25
+ <div class="runs" id="live"></div>
26
+ </section>
27
+
28
+ <section style="margin-bottom:32px">
29
+ <div class="sect-h" style="margin-bottom:12px">
30
+ <h2>Best loss by size</h2>
31
+ <span class="sub">lower is better</span>
32
+ </div>
33
+ <div id="board" class="grid g2"></div>
34
+ </section>
35
+
36
+ <section style="margin-bottom:32px">
37
+ <div class="sect-h" style="margin-bottom:12px"><h2>All models</h2></div>
38
+ <div style="display:flex;gap:7px;flex-wrap:wrap;margin-bottom:8px" id="tierFilter">
39
+ <button class="btn sm primary" data-tier="">All sizes</button>
40
+ {% for t in tiers %}
41
+ <button class="btn sm" data-tier="{{ t.key }}">{{ t.label }}</button>
42
+ {% endfor %}
43
+ </div>
44
+ <div style="display:flex;gap:7px;flex-wrap:wrap;margin-bottom:18px" id="sortFilter">
45
+ <button class="btn sm primary" data-sort="recent">Newest</button>
46
+ <button class="btn sm" data-sort="loss">Lowest loss</button>
47
+ <button class="btn sm" data-sort="tokens">Most tokens</button>
48
+ <button class="btn sm" data-sort="params">Largest</button>
49
+ </div>
50
+ <div id="models"></div>
51
+ </section>
52
+
53
+ <section>
54
+ <div class="sect-h" style="margin-bottom:12px"><h2>Trainers</h2></div>
55
+ <div id="trainers" class="grid g3"></div>
56
+ </section>
57
+
58
+ {% endblock %}
59
+ {% block scripts %}
60
+ <script>
61
+ let tier = '', sort = 'recent';
62
+
63
+ function card(m){
64
+ return `<a class="run" href="/models/${m.id}">
65
+ <div class="run-top">
66
+ <h4>${NX.esc(m.name)}</h4>
67
+ ${NX.chip(m.status)}
68
+ </div>
69
+ <div class="run-meta">
70
+ <span class="mono">${NX.esc(m.tier_label)}</span>
71
+ <span>${Number(m.n_params).toLocaleString()} params</span>
72
+ <span><b>${NX.fmtTokens(m.tokens_seen)}</b> tokens</span>
73
+ ${m.best_loss ? `<span>loss <b>${m.best_loss.toFixed(4)}</b></span>` : ''}
74
+ <span>@${NX.esc(m.username)}</span>
75
+ <span class="muted">${NX.ago(m.finished_at || m.created_at)}</span>
76
+ </div>
77
+ </a>`;
78
+ }
79
+
80
+ function bind(sel, key, fn){
81
+ document.querySelectorAll(sel + ' .btn').forEach(b =>
82
+ b.addEventListener('click', () => {
83
+ document.querySelectorAll(sel + ' .btn').forEach(x => x.classList.remove('primary'));
84
+ b.classList.add('primary');
85
+ fn(b.dataset[key]);
86
+ loadModels();
87
+ }));
88
+ }
89
+ bind('#tierFilter', 'tier', v => tier = v);
90
+ bind('#sortFilter', 'sort', v => sort = v);
91
+
92
+ async function loadModels(){
93
+ const q = await NX.get(`/api/explore?tier=${tier}&sort=${sort}&limit=60`);
94
+
95
+ document.getElementById('s-done').textContent = q.stats.done;
96
+ document.getElementById('s-run').textContent = q.stats.running;
97
+ document.getElementById('s-users').textContent = q.stats.users;
98
+ document.getElementById('s-tok').textContent = NX.fmtTokens(q.stats.tokens);
99
+
100
+ const liveSec = document.getElementById('liveSec');
101
+ liveSec.style.display = q.active.length ? 'block' : 'none';
102
+ document.getElementById('live').innerHTML = q.active.map(NX.runRow).join('');
103
+
104
+ document.getElementById('models').innerHTML = q.models.length
105
+ ? `<div class="runs">${q.models.map(card).join('')}</div>`
106
+ : `<div class="empty">No finished models${tier ? ' at this size' : ''} yet.</div>`;
107
+ }
108
+
109
+ async function loadBoard(){
110
+ const {tiers} = await NX.get('/api/leaderboard');
111
+ const el = document.getElementById('board');
112
+ if(!tiers.length){
113
+ el.innerHTML = `<div class="empty" style="grid-column:1/-1">
114
+ Nothing finished yet — the first model trained here takes every record.</div>`;
115
+ return;
116
+ }
117
+ el.innerHTML = tiers.map(t => `<div class="card">
118
+ <div class="eyebrow" style="margin-bottom:10px">${NX.esc(t.label)}</div>
119
+ ${t.models.map((m,i) => `<div class="kv">
120
+ <span><b class="mono" style="color:var(--accent)">#${i+1}</b>
121
+ <a href="/models/${m.id}" style="color:var(--fg)">${NX.esc(m.name)}</a>
122
+ <span class="muted">@${NX.esc(m.username)}</span></span>
123
+ <b>${m.best_loss.toFixed(4)}</b></div>`).join('')}
124
+ </div>`).join('');
125
+ }
126
+
127
+ async function loadTrainers(){
128
+ const {trainers} = await NX.get('/api/trainers');
129
+ document.getElementById('trainers').innerHTML = trainers.length
130
+ ? trainers.map(t => `<a class="card" href="/u/${encodeURIComponent(t.username)}"
131
+ style="display:flex;gap:12px;align-items:center">
132
+ ${t.avatar_url
133
+ ? `<img class="avatar" style="width:40px;height:40px" src="${NX.esc(t.avatar_url)}" alt="">`
134
+ : `<span class="avatar" style="width:40px;height:40px"></span>`}
135
+ <div style="min-width:0">
136
+ <div style="font-weight:620">${NX.esc(t.display_name || t.username)}</div>
137
+ <div class="sub" style="font-size:12.5px">
138
+ ${t.done} model${t.done === 1 ? '' : 's'} ·
139
+ ${NX.fmtTokens(t.tokens)} tokens${t.best_loss
140
+ ? ' · best ' + t.best_loss.toFixed(3) : ''}</div>
141
+ </div></a>`).join('')
142
+ : `<div class="empty" style="grid-column:1/-1">No trainers yet.</div>`;
143
+ }
144
+
145
+ NX.poll(async () => { await loadModels(); await loadBoard(); await loadTrainers(); }, 8000);
146
+ </script>
147
+ {% endblock %}
web/templates/home.html ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'home' %}
3
+ {% block title %}NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div class="sect-h" style="margin-bottom:20px">
7
+ <div>
8
+ <span class="eyebrow">Welcome back</span>
9
+ <h1 style="font-size:28px;margin-top:5px">{{ user.name or user.username }}</h1>
10
+ </div>
11
+ <a class="btn primary" href="/train">+ New run</a>
12
+ </div>
13
+
14
+ <div class="grid g4" id="tiles">
15
+ <div class="stat"><span>Your runs</span><b id="t-mine">—</b><small id="t-mine-s">&nbsp;</small></div>
16
+ <div class="stat"><span>In flight</span><b id="t-active">—</b><small id="t-active-s">&nbsp;</small></div>
17
+ <div class="stat"><span>Tokens trained</span><b id="t-tok">—</b><small>across all your runs</small></div>
18
+ <div class="stat"><span>Best loss</span><b id="t-loss">—</b><small id="t-loss-s">&nbsp;</small></div>
19
+ </div>
20
+
21
+ <section style="margin-top:34px">
22
+ <div class="sect-h">
23
+ <h2>Live now</h2>
24
+ <a class="btn ghost sm" href="/queue">Global queue →</a>
25
+ </div>
26
+ <div id="live" class="runs"></div>
27
+ </section>
28
+
29
+ <section style="margin-top:34px">
30
+ <div class="sect-h">
31
+ <h2>Recent runs</h2>
32
+ <a class="btn ghost sm" href="/models">All models →</a>
33
+ </div>
34
+ <div id="recent" class="runs"></div>
35
+ </section>
36
+
37
+ {% endblock %}
38
+ {% block scripts %}
39
+ <script>
40
+ async function refresh(){
41
+ const {jobs} = await NX.get('/api/jobs');
42
+ const active = jobs.filter(j => j.status==='running' || j.status==='queued');
43
+ const done = jobs.filter(j => j.status==='done');
44
+
45
+ document.getElementById('t-mine').textContent = jobs.length;
46
+ document.getElementById('t-mine-s').textContent = done.length + ' finished';
47
+ document.getElementById('t-active').textContent = active.length;
48
+ document.getElementById('t-active-s').textContent =
49
+ active.filter(j=>j.status==='running').length + ' training now';
50
+ document.getElementById('t-tok').textContent =
51
+ NX.fmtTokens(jobs.reduce((a,j)=>a+(j.tokens_seen||0),0));
52
+ const best = done.map(j=>j.best_loss).filter(Boolean);
53
+ document.getElementById('t-loss').textContent =
54
+ best.length ? Math.min(...best).toFixed(4) : '—';
55
+ document.getElementById('t-loss-s').textContent =
56
+ best.length ? 'cross-entropy' : 'no finished runs yet';
57
+
58
+ const live = document.getElementById('live');
59
+ live.innerHTML = active.length
60
+ ? active.map(NX.runRow).join('')
61
+ : `<div class="empty">Nothing of yours is training right now.
62
+ <a href="/train" style="color:var(--accent)">Start a run →</a></div>`;
63
+
64
+ const rec = jobs.filter(j => j.status!=='running' && j.status!=='queued').slice(0,6);
65
+ document.getElementById('recent').innerHTML = rec.length
66
+ ? rec.map(NX.runRow).join('')
67
+ : `<div class="empty">No finished runs yet — your first model is one click away.</div>`;
68
+ }
69
+ NX.poll(refresh, 4000);
70
+ </script>
71
+ {% endblock %}
web/templates/landing.html ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% block title %}NanoDex — train a language model from scratch{% endblock %}
3
+ {% block body %}
4
+
5
+ <section class="hero">
6
+ <span class="tag"><b>{{ n_devices }}</b> {{ 'GPU' if on_gpu else 'CPU' }} worker{{ 's' if n_devices != 1 }} ·
7
+ <b>{{ stats.done }}</b> models born here</span>
8
+ <h1>Train your first<br><em>decoder-only model</em><br>from scratch.</h1>
9
+ <p>Not a fine-tune. Not an adapter. Random weights, a real corpus, and a real
10
+ loss curve you watch fall in real time — from a browser tab, on our GPUs.</p>
11
+ <div class="cta">
12
+ <a class="btn primary lg" target="_top" href="/login?next=/train">Start a run &nbsp;→</a>
13
+ <a class="btn lg" href="/explore">Browse the models</a>
14
+ </div>
15
+ <p class="frame-note">Signing in opens NanoDex full screen — Hugging Face's
16
+ login page can't run inside an embedded frame.</p>
17
+ </section>
18
+
19
+ <section style="margin-top:22px">
20
+ <div class="grid g3">
21
+ <div class="card">
22
+ <div class="step-n">1</div>
23
+ <h3>Pick a size</h3>
24
+ <p class="sub" style="margin:6px 0 0">Four architectures, from 492k to 8.1M
25
+ parameters. That's the <em>real</em> total — embeddings included, no asterisk.</p>
26
+ </div>
27
+ <div class="card">
28
+ <div class="step-n">2</div>
29
+ <h3>Pick a token budget</h3>
30
+ <p class="sub" style="margin:6px 0 0">Anywhere from
31
+ {{ min_tokens // 10**6 }}M to <b>1.5 billion</b> tokens of
32
+ <b>fineweb-edu</b>. More tokens, sharper model, longer wait.</p>
33
+ </div>
34
+ <div class="card">
35
+ <div class="step-n">3</div>
36
+ <h3>Watch it learn</h3>
37
+ <p class="sub" style="margin:6px 0 0">Your run joins the queue. Live loss
38
+ curve, throughput, logs — then the weights land in your own Hugging Face
39
+ account.</p>
40
+ </div>
41
+ </div>
42
+ </section>
43
+
44
+ <section style="margin-top:44px">
45
+ <div class="sect-h">
46
+ <div>
47
+ <span class="eyebrow">The four sizes</span>
48
+ <h2 style="margin-top:5px">Small enough to finish. Big enough to learn something.</h2>
49
+ </div>
50
+ </div>
51
+ <div class="picks">
52
+ {% for t in tiers %}
53
+ <div class="pick">
54
+ <b>{{ t.label }}</b>
55
+ <div class="p-n">{{ "{:,}".format(t.params) }} params</div>
56
+ <div class="p-d">
57
+ {{ t.layers }} layers · {{ t.hidden }} hidden<br>
58
+ {{ t.heads }} heads ({{ t.kv_heads }} KV) · FFN {{ t.ffn }}
59
+ </div>
60
+ </div>
61
+ {% endfor %}
62
+ </div>
63
+ </section>
64
+
65
+ <section style="margin-top:44px">
66
+ <div class="grid g2">
67
+ <div class="card pad-lg">
68
+ <span class="eyebrow">What you actually get</span>
69
+ <h2 style="margin:8px 0 12px;font-size:21px">A model that is genuinely yours</h2>
70
+ <p class="sub">Every finished run is pushed to <b>your</b> Hugging Face
71
+ namespace: <span class="mono">config.json</span>,
72
+ <span class="mono">model.safetensors</span>, the tokenizer, a
73
+ <span class="mono">training_run.json</span> with the full recipe, and a
74
+ generated model card. Load it with <span class="mono">transformers</span>
75
+ like anything else on the Hub.</p>
76
+ </div>
77
+ <div class="card pad-lg">
78
+ <span class="eyebrow">Honest expectations</span>
79
+ <h2 style="margin:8px 0 12px;font-size:21px">It will not be an assistant</h2>
80
+ <p class="sub">At a few hundred thousand parameters, a model learns word
81
+ shapes, common collocations and a little syntax. It will produce
82
+ English-looking text with no facts in it. That's the correct outcome —
83
+ and watching cross-entropy fall from <b>7.6</b> (uniform noise over 2,048
84
+ tokens) to somewhere near <b>4</b> is the whole point.</p>
85
+ </div>
86
+ </div>
87
+ </section>
88
+
89
+ {% if recent %}
90
+ <section style="margin-top:44px">
91
+ <div class="sect-h">
92
+ <div>
93
+ <span class="eyebrow">Trained here</span>
94
+ <h2 style="margin-top:5px">The newest models on the shelf</h2>
95
+ </div>
96
+ <a class="btn ghost sm" href="/explore">Explore all →</a>
97
+ </div>
98
+ <div class="runs">
99
+ {% for m in recent %}
100
+ <a class="run" href="/models/{{ m.id }}">
101
+ <div class="run-top">
102
+ <h4>{{ m.name }}</h4>
103
+ <span class="chip done"><i></i>completed</span>
104
+ </div>
105
+ <div class="run-meta">
106
+ <span class="mono">{{ m.tier_label }}</span>
107
+ <span>{{ "{:,}".format(m.n_params) }} params</span>
108
+ <span><b>{{ (m.tokens_seen / 1000000)|round|int }}M</b> tokens</span>
109
+ {% if m.best_loss %}<span>loss <b>{{ "%.4f"|format(m.best_loss) }}</b></span>{% endif %}
110
+ <span>@{{ m.username }}</span>
111
+ </div>
112
+ </a>
113
+ {% endfor %}
114
+ </div>
115
+ </section>
116
+ {% endif %}
117
+
118
+ <section style="margin-top:44px" class="card pad-lg">
119
+ <div style="display:flex;gap:24px;align-items:center;justify-content:space-between;flex-wrap:wrap">
120
+ <div>
121
+ <h2 style="font-size:21px">Ready when you are.</h2>
122
+ <p class="sub" style="margin-top:6px">Sign in with Hugging Face — no separate account, no password.</p>
123
+ </div>
124
+ <a class="btn primary lg" target="_top" href="/login?next=/train">Start a run &nbsp;→</a>
125
+ </div>
126
+ </section>
127
+
128
+ {% endblock %}
web/templates/models.html ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'models' %}
3
+ {% block title %}Models — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div class="sect-h" style="margin-bottom:22px">
7
+ <div>
8
+ <span class="eyebrow">Your models</span>
9
+ <h1 style="font-size:28px;margin-top:5px">Everything you've trained</h1>
10
+ </div>
11
+ <a class="btn primary" href="/train">+ New run</a>
12
+ </div>
13
+
14
+ <div style="display:flex;gap:7px;flex-wrap:wrap;margin-bottom:22px" id="filters">
15
+ <button class="btn sm on" data-f="all">All</button>
16
+ <button class="btn sm" data-f="running">Training</button>
17
+ <button class="btn sm" data-f="queued">Queued</button>
18
+ <button class="btn sm" data-f="done">Completed</button>
19
+ <button class="btn sm" data-f="other">Failed / cancelled</button>
20
+ </div>
21
+
22
+ <section id="sec-running" style="margin-bottom:32px"></section>
23
+ <section id="sec-queued" style="margin-bottom:32px"></section>
24
+ <section id="sec-done" style="margin-bottom:32px"></section>
25
+ <section id="sec-other"></section>
26
+ <div id="none"></div>
27
+
28
+ {% endblock %}
29
+ {% block scripts %}
30
+ <script>
31
+ let filter = 'all';
32
+
33
+ document.querySelectorAll('#filters .btn').forEach(b =>
34
+ b.addEventListener('click', () => {
35
+ document.querySelectorAll('#filters .btn').forEach(x =>
36
+ x.classList.remove('primary'));
37
+ b.classList.add('primary');
38
+ filter = b.dataset.f;
39
+ render(last);
40
+ }));
41
+ document.querySelector('#filters .btn').classList.add('primary');
42
+
43
+ let last = [];
44
+
45
+ function section(id, title, note, jobs){
46
+ const el = document.getElementById(id);
47
+ if(!jobs.length){ el.innerHTML = ''; return 0; }
48
+ el.innerHTML = `<div class="sect-h">
49
+ <div><h2>${title} <span class="muted" style="font-weight:400">
50
+ (${jobs.length})</span></h2>
51
+ ${note ? `<p class="sub" style="margin-top:3px">${note}</p>` : ''}</div>
52
+ </div>
53
+ <div class="runs">${jobs.map(NX.runRow).join('')}</div>`;
54
+ return jobs.length;
55
+ }
56
+
57
+ function render(jobs){
58
+ last = jobs;
59
+ const pick = s => (filter==='all' || filter===s) ? jobs.filter(j =>
60
+ s==='other' ? (j.status==='failed'||j.status==='cancelled') : j.status===s) : [];
61
+
62
+ let n = 0;
63
+ n += section('sec-running','Training now','Live on a worker right now.', pick('running'));
64
+ n += section('sec-queued','In the queue','Waiting for a free worker — oldest first.', pick('queued'));
65
+ n += section('sec-done','Completed','Finished. Open one to publish it or generate text.', pick('done'));
66
+ n += section('sec-other','Failed or cancelled','', pick('other'));
67
+
68
+ document.getElementById('none').innerHTML = n ? '' :
69
+ `<div class="empty">Nothing here yet.
70
+ <a href="/train" style="color:var(--accent)">Train your first model →</a></div>`;
71
+ }
72
+
73
+ NX.poll(async () => render((await NX.get('/api/jobs')).jobs), 4000);
74
+ </script>
75
+ {% endblock %}
web/templates/profile.html ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% block title %}{{ user.username }} — NanoDex{% endblock %}
3
+ {% block body %}
4
+
5
+ <div class="card pad-lg" style="display:flex;gap:20px;align-items:center;flex-wrap:wrap">
6
+ {% if user.picture %}
7
+ <img src="{{ user.picture }}" alt="" style="width:72px;height:72px;border-radius:50%;
8
+ border:1px solid var(--line-2);object-fit:cover">
9
+ {% else %}
10
+ <div style="width:72px;height:72px;border-radius:50%;background:var(--panel-2);
11
+ border:1px solid var(--line-2)"></div>
12
+ {% endif %}
13
+ <div style="flex:1;min-width:200px">
14
+ <h1 style="font-size:24px">{{ user.name or user.username }}</h1>
15
+ <p class="sub" style="margin-top:3px">
16
+ <a href="https://huggingface.co/{{ user.username }}" target="_blank" rel="noopener"
17
+ class="mono" style="color:var(--fg-2)">@{{ user.username }}</a>
18
+ · signed in with Hugging Face
19
+ </p>
20
+ </div>
21
+ <div style="display:flex;gap:8px;flex-wrap:wrap">
22
+ <a class="btn" href="/u/{{ user.username }}">Public profile</a>
23
+ <a class="btn" href="/models">My models</a>
24
+ <a class="btn ghost" href="/logout">Sign out</a>
25
+ </div>
26
+ </div>
27
+
28
+ <div class="grid g4" style="margin-top:16px">
29
+ <div class="stat"><span>Runs</span><b>{{ summary.total }}</b>
30
+ <small>{{ summary.done }} finished · {{ summary.active }} in flight</small></div>
31
+ <div class="stat"><span>Tokens trained</span>
32
+ <b>{{ "%.0fM"|format(summary.tokens / 1000000) if summary.tokens < 1000000000 else "%.2fB"|format(summary.tokens / 1000000000) }}</b>
33
+ <small>of fineweb-edu</small></div>
34
+ <div class="stat"><span>Parameters created</span>
35
+ <b>{{ "{:,}".format(summary.params) }}</b><small>across finished models</small></div>
36
+ <div class="stat"><span>Best loss</span>
37
+ <b>{{ "%.4f"|format(summary.best) if summary.best else "—" }}</b>
38
+ <small>{{ summary.published }} published to the Hub</small></div>
39
+ </div>
40
+
41
+ <section style="margin-top:34px">
42
+ <div class="sect-h">
43
+ <h2>Recent activity</h2>
44
+ <a class="btn ghost sm" href="/models">All models →</a>
45
+ </div>
46
+ <div id="runs" class="runs"></div>
47
+ </section>
48
+
49
+ <section style="margin-top:34px">
50
+ <div class="card">
51
+ <h2 style="font-size:16px;margin-bottom:6px">Permissions</h2>
52
+ <p class="sub">NanoDex uses your Hugging Face sign-in for two things: to know
53
+ which runs are yours, and — only when you press Publish — to upload a
54
+ finished model to your namespace. Nothing is written to your account
55
+ otherwise, and the session expires after 8 hours.</p>
56
+ <p class="sub" style="margin-top:12px">Your finished models are visible to
57
+ everyone who can open NanoDex, on your
58
+ <a href="/u/{{ user.username }}" style="color:var(--accent)">public
59
+ profile</a> and in <a href="/explore" style="color:var(--accent)">Explore</a>,
60
+ and anyone signed in can generate text from them. Cancelling a run and
61
+ publishing to the Hub stay yours alone.</p>
62
+ </div>
63
+ </section>
64
+
65
+ {% endblock %}
66
+ {% block scripts %}
67
+ <script>
68
+ const JOBS = {{ jobs | tojson }};
69
+ document.getElementById('runs').innerHTML = JOBS.length
70
+ ? JOBS.slice(0,10).map(NX.runRow).join('')
71
+ : `<div class="empty">No runs yet.
72
+ <a href="/train" style="color:var(--accent)">Train your first model →</a></div>`;
73
+ </script>
74
+ {% endblock %}
web/templates/queue.html ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'queue' %}
3
+ {% block title %}Queue — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div class="sect-h" style="margin-bottom:22px">
7
+ <div>
8
+ <span class="eyebrow">Shared infrastructure</span>
9
+ <h1 style="font-size:28px;margin-top:5px">The queue, right now</h1>
10
+ </div>
11
+ {% if user %}<a class="btn primary" href="/train">+ New run</a>
12
+ {% else %}<a class="btn primary" target="_top" href="/login?next=/train">Start a run</a>{% endif %}
13
+ </div>
14
+
15
+ <div class="grid g4" style="margin-bottom:20px">
16
+ <div class="stat"><span>Training</span><b id="s-run">—</b><small>on workers</small></div>
17
+ <div class="stat"><span>Waiting</span><b id="s-q">—</b><small>in the queue</small></div>
18
+ <div class="stat"><span>Models born</span><b id="s-done">—</b><small>finished runs</small></div>
19
+ <div class="stat"><span>Tokens burned</span><b id="s-tok">—</b><small>all time</small></div>
20
+ </div>
21
+
22
+ <section style="margin-bottom:30px">
23
+ <div class="sect-h"><h2>Workers</h2></div>
24
+ <div class="grid g3" id="workers"></div>
25
+ <p class="hint" id="cache"></p>
26
+ <p class="hint" id="storage"></p>
27
+ </section>
28
+
29
+ <section style="margin-bottom:30px">
30
+ <div class="sect-h"><h2>Waiting in line</h2></div>
31
+ <div id="queued"></div>
32
+ </section>
33
+
34
+ <section>
35
+ <div class="sect-h"><h2>Recently finished</h2></div>
36
+ <div id="recent"></div>
37
+ </section>
38
+
39
+ {% endblock %}
40
+ {% block scripts %}
41
+ <script>
42
+ function tbl(rows, cols){
43
+ if(!rows.length) return '';
44
+ return `<div class="card tscroll" style="padding:6px 4px 4px"><table>
45
+ <thead><tr>${cols.map(c=>`<th>${c[0]}</th>`).join('')}</tr></thead>
46
+ <tbody>${rows.map(r=>`<tr>${cols.map(c=>`<td>${c[1](r)}</td>`).join('')}</tr>`).join('')}</tbody>
47
+ </table></div>`;
48
+ }
49
+
50
+ async function refresh(){
51
+ const q = await NX.get('/api/queue');
52
+
53
+ document.getElementById('s-run').textContent = q.stats.running;
54
+ document.getElementById('s-q').textContent = q.stats.queued;
55
+ document.getElementById('s-done').textContent = q.stats.done;
56
+ document.getElementById('s-tok').textContent = NX.fmtTokens(q.stats.tokens);
57
+
58
+ document.getElementById('workers').innerHTML = q.workers.map(w => {
59
+ if(!w.job){
60
+ return `<div class="gpu">
61
+ <div class="gpu-h"><b>Worker ${w.index}</b><span>${NX.esc(w.device)}</span></div>
62
+ <div class="muted" style="font-size:13.5px">idle — ready for the next run</div>
63
+ </div>`;
64
+ }
65
+ const j = w.job, pct = (j.progress*100).toFixed(1);
66
+ return `<div class="gpu busy">
67
+ <div class="gpu-h"><b>Worker ${w.index}</b><span>${NX.esc(w.device)}</span></div>
68
+ <div style="font-weight:620;font-size:14.5px">${NX.esc(j.name)}</div>
69
+ <div class="sub" style="font-size:12.5px">@${NX.esc(j.username)} · ${NX.esc(j.tier_label)}</div>
70
+ <div class="bar thin" style="margin:10px 0 7px"><i style="width:${pct}%"></i></div>
71
+ <div class="mono muted" style="font-size:12px">${pct}% ·
72
+ loss ${j.loss ? j.loss.toFixed(3) : '—'} ·
73
+ ${Math.round(j.tok_per_s/1e3)}k tok/s</div>
74
+ </div>`;
75
+ }).join('');
76
+
77
+ const d = q.data;
78
+ const pct = Math.min(100, 100 * d.ready_tokens / (d.target||1));
79
+ document.getElementById('cache').textContent =
80
+ `fineweb-edu token cache: ${NX.fmtTokens(d.ready_tokens)} tokens ready ` +
81
+ `(${pct.toFixed(0)}% · ${d.status})`;
82
+
83
+ const st = q.store || {};
84
+ document.getElementById('storage').innerHTML = st.enabled
85
+ ? `durable storage: <b>on</b> — finished models and their loss curves are
86
+ archived to <span class="mono">${NX.esc(st.bucket)}</span> and survive
87
+ restarts${st.restored ? ` (${st.restored} run${st.restored===1?'':'s'} restored on boot)` : ''}`
88
+ : `durable storage: <b style="color:var(--amber)">off</b> — this container's
89
+ history and checkpoints are lost on the next restart`;
90
+
91
+ document.getElementById('queued').innerHTML = q.queued.length
92
+ ? tbl(q.queued, [
93
+ ['#', r => '<b>'+(r.queue_position||'?')+'</b>'],
94
+ ['Model', r => NX.esc(r.name)],
95
+ ['Author', r => `<a href="/u/${encodeURIComponent(r.username)}"
96
+ style="color:var(--accent)">@${NX.esc(r.username)}</a>`],
97
+ ['Size', r => NX.esc(r.tier_label)],
98
+ ['Tokens', r => NX.fmtTokens(r.target_tokens)],
99
+ ['Waiting', r => NX.ago(r.created_at)],
100
+ ])
101
+ : `<div class="empty">Queue is empty — a worker will start your run immediately.</div>`;
102
+
103
+ document.getElementById('recent').innerHTML = q.recent.length
104
+ ? tbl(q.recent, [
105
+ ['Model', r => NX.esc(r.name)],
106
+ ['Author', r => `<a href="/u/${encodeURIComponent(r.username)}"
107
+ style="color:var(--accent)">@${NX.esc(r.username)}</a>`],
108
+ ['Size', r => NX.esc(r.tier_label)],
109
+ ['Tokens', r => NX.fmtTokens(r.tokens_seen)],
110
+ ['Final loss', r => r.loss ? r.loss.toFixed(4) : '—'],
111
+ ['On the Hub', r => r.pushed_repo
112
+ ? `<a class="mono" style="color:var(--accent)" target="_blank" rel="noopener"
113
+ href="https://huggingface.co/${NX.esc(r.pushed_repo)}">${NX.esc(r.pushed_repo)}</a>`
114
+ : '<span class="muted">not published</span>'],
115
+ ])
116
+ : `<div class="empty">No finished runs yet. Someone has to be first.</div>`;
117
+ }
118
+ NX.poll(refresh, 4000);
119
+ </script>
120
+ {% endblock %}
web/templates/train.html ADDED
@@ -0,0 +1,273 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'train' %}
3
+ {% block title %}New run — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div style="margin-bottom:24px">
7
+ <span class="eyebrow">New training run</span>
8
+ <h1 style="font-size:28px;margin-top:5px">Build a model from nothing</h1>
9
+ </div>
10
+
11
+ <div class="steps">
12
+ <div class="stp on" data-s="1"><div class="n">Step 1</div><div class="t">Size</div></div>
13
+ <div class="stp" data-s="2"><div class="n">Step 2</div><div class="t">Tokens</div></div>
14
+ <div class="stp" data-s="3"><div class="n">Step 3</div><div class="t">Name</div></div>
15
+ <div class="stp" data-s="4"><div class="n">Step 4</div><div class="t">Review</div></div>
16
+ </div>
17
+
18
+ <!-- STEP 1 ---------------------------------------------------------------->
19
+ <div class="panelstep on" data-p="1">
20
+ <h2 style="font-size:19px">How big should it be?</h2>
21
+ <p class="sub" style="margin:6px 0 20px">Every tier is the same architecture —
22
+ a <span class="mono">LlamaForCausalLM</span> decoder-only transformer with
23
+ SiLU MLPs, RMSNorm, rotary embeddings and grouped-query attention — scaled
24
+ down in width and depth. The parameter counts below are totals, embeddings
25
+ included.</p>
26
+
27
+ <div class="picks">
28
+ {% for t in tiers %}
29
+ <button type="button" class="pick {{ 'on' if t.key == '1m' }}" data-tier="{{ t.key }}"
30
+ data-params="{{ t.params }}" data-label="{{ t.label }}">
31
+ <b>{{ t.label }}</b>
32
+ <div class="p-n">{{ "{:,}".format(t.params) }} params</div>
33
+ <div class="p-d">
34
+ {{ t.layers }} layers · {{ t.hidden }} hidden<br>
35
+ {{ t.heads }} heads ({{ t.kv_heads }} KV)<br>
36
+ FFN {{ t.ffn }} · ctx {{ seq_len }}
37
+ </div>
38
+ </button>
39
+ {% endfor %}
40
+ </div>
41
+
42
+ <div style="display:flex;justify-content:flex-end;margin-top:24px">
43
+ <button class="btn primary" data-go="2">Continue &nbsp;→</button>
44
+ </div>
45
+ </div>
46
+
47
+ <!-- STEP 2 ---------------------------------------------------------------->
48
+ <div class="panelstep" data-p="2">
49
+ <h2 style="font-size:19px">How much should it read?</h2>
50
+ <p class="sub" style="margin:6px 0 24px">Training data is
51
+ <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu" target="_blank"
52
+ rel="noopener" style="color:var(--accent)">fineweb-edu</a> — filtered
53
+ educational web text. More tokens means a sharper model and a longer wait.</p>
54
+
55
+ <div class="card pad-lg">
56
+ <div style="display:flex;justify-content:space-between;align-items:baseline;margin-bottom:16px">
57
+ <label class="fl" style="margin:0">Token budget</label>
58
+ <div class="mono" style="font-size:26px;font-weight:680;color:var(--accent)"
59
+ id="tokLabel">500M</div>
60
+ </div>
61
+ <input type="range" id="tok" min="0" max="{{ stops|length - 1 }}"
62
+ value="4" step="1">
63
+ <div class="mono muted"
64
+ style="display:flex;justify-content:space-between;margin-top:9px;font-size:12px">
65
+ <span>200M</span><span>500M</span><span>1B</span><span>1.5B</span>
66
+ </div>
67
+ <div class="quickpick">
68
+ {% for m in [200, 500, 1000, 1500] %}
69
+ <button type="button" class="btn sm" data-tokens="{{ m }}">
70
+ {{ (m / 1000)|round(1)|string|replace('.0','') ~ 'B' if m >= 1000
71
+ else (m|string ~ 'M') }}</button>
72
+ {% endfor %}
73
+ </div>
74
+ </div>
75
+
76
+ <div class="grid g4" style="margin-top:16px">
77
+ <div class="stat"><span>Parameters</span><b id="e-params">—</b><small id="e-tier">&nbsp;</small></div>
78
+ <div class="stat"><span>Optimizer steps</span><b id="e-steps">—</b><small>gradient updates</small></div>
79
+ <div class="stat"><span>Estimated time</span><b id="e-eta">—</b><small id="e-cal">&nbsp;</small></div>
80
+ <div class="stat"><span>Tokens / param</span><b id="e-ratio">—</b><small id="e-chin">&nbsp;</small></div>
81
+ </div>
82
+
83
+ <p class="hint" id="scaleNote"></p>
84
+
85
+ <div style="display:flex;justify-content:space-between;margin-top:24px">
86
+ <button class="btn ghost" data-go="1">← Back</button>
87
+ <button class="btn primary" data-go="3">Continue &nbsp;→</button>
88
+ </div>
89
+ </div>
90
+
91
+ <!-- STEP 3 ---------------------------------------------------------------->
92
+ <div class="panelstep" data-p="3">
93
+ <h2 style="font-size:19px">Give it a name</h2>
94
+ <p class="sub" style="margin:6px 0 20px">This is what you'll see in your model
95
+ list, and the default repository name when you publish it.</p>
96
+
97
+ <div class="card pad-lg" style="max-width:560px">
98
+ <label class="fl" for="name">Model name</label>
99
+ <input type="text" id="name" placeholder="my-first-lm" maxlength="80" autocomplete="off">
100
+ <p class="hint">Letters, numbers, <span class="mono">-</span>,
101
+ <span class="mono">_</span> and <span class="mono">.</span>. Leave it blank
102
+ and we'll name it after the size and token budget.</p>
103
+ <p class="hint" style="margin-top:14px">Will publish to
104
+ <span class="mono" style="color:var(--fg-2)">{{ user.username }}/<span id="namePreview">…</span></span></p>
105
+ </div>
106
+
107
+ <div style="display:flex;justify-content:space-between;margin-top:24px">
108
+ <button class="btn ghost" data-go="2">← Back</button>
109
+ <button class="btn primary" data-go="4">Review &nbsp;→</button>
110
+ </div>
111
+ </div>
112
+
113
+ <!-- STEP 4 ---------------------------------------------------------------->
114
+ <div class="panelstep" data-p="4">
115
+ <h2 style="font-size:19px">Ready to launch</h2>
116
+ <p class="sub" style="margin:6px 0 20px">Your run joins the shared queue. You
117
+ can watch it — or close the tab and come back; it keeps going either way.</p>
118
+
119
+ <div class="grid g2">
120
+ <div class="card pad-lg">
121
+ <span class="eyebrow">Model</span>
122
+ <div style="margin-top:12px">
123
+ <div class="kv"><span>Name</span><b id="r-name">—</b></div>
124
+ <div class="kv"><span>Size</span><b id="r-tier">—</b></div>
125
+ <div class="kv"><span>Parameters</span><b id="r-params">—</b></div>
126
+ <div class="kv"><span>Architecture</span><b id="r-arch">—</b></div>
127
+ <div class="kv"><span>Vocabulary</span><b>{{ "{:,}".format(vocab) }} BPE</b></div>
128
+ <div class="kv"><span>Context</span><b>{{ seq_len }} tokens</b></div>
129
+ </div>
130
+ </div>
131
+ <div class="card pad-lg">
132
+ <span class="eyebrow">Training</span>
133
+ <div style="margin-top:12px">
134
+ <div class="kv"><span>Dataset</span><b>fineweb-edu</b></div>
135
+ <div class="kv"><span>Token budget</span><b id="r-tok">—</b></div>
136
+ <div class="kv"><span>Optimizer steps</span><b id="r-steps">—</b></div>
137
+ <div class="kv"><span>Optimizer</span><b>AdamW · cosine</b></div>
138
+ <div class="kv"><span>Estimated time</span><b id="r-eta">—</b></div>
139
+ <div class="kv"><span>Queue limit</span><b>{{ max_active }} runs per person</b></div>
140
+ </div>
141
+ </div>
142
+ </div>
143
+
144
+ <div style="display:flex;justify-content:space-between;margin-top:24px;gap:12px;flex-wrap:wrap">
145
+ <button class="btn ghost" data-go="3">← Back</button>
146
+ <button class="btn primary lg" id="launch">Queue this run &nbsp;→</button>
147
+ </div>
148
+ </div>
149
+
150
+ {% endblock %}
151
+ {% block scripts %}
152
+ <script>
153
+ const STOPS = {{ stops | tojson }};
154
+ const state = { tier:'1m', tokens:500, name:'' };
155
+
156
+ function labelTokens(m){
157
+ return m >= 1000 ? String(m/1000).replace(/\.0$/,'') + 'B' : m + 'M';
158
+ }
159
+ let est = null;
160
+
161
+ function showStep(n){
162
+ document.querySelectorAll('.panelstep').forEach(p =>
163
+ p.classList.toggle('on', p.dataset.p === String(n)));
164
+ document.querySelectorAll('.stp').forEach(s => {
165
+ const i = Number(s.dataset.s);
166
+ s.classList.toggle('on', i === n);
167
+ s.classList.toggle('ok', i < n);
168
+ });
169
+ window.scrollTo({top:0, behavior:'smooth'});
170
+ if(n === 4) fillReview();
171
+ }
172
+
173
+ document.querySelectorAll('[data-go]').forEach(b =>
174
+ b.addEventListener('click', () => showStep(Number(b.dataset.go))));
175
+ document.querySelectorAll('.stp').forEach(s =>
176
+ s.addEventListener('click', () => showStep(Number(s.dataset.s))));
177
+
178
+ document.querySelectorAll('.pick[data-tier]').forEach(p =>
179
+ p.addEventListener('click', () => {
180
+ document.querySelectorAll('.pick[data-tier]').forEach(q => q.classList.remove('on'));
181
+ p.classList.add('on');
182
+ state.tier = p.dataset.tier;
183
+ updateEstimate();
184
+ }));
185
+
186
+ const tok = document.getElementById('tok');
187
+ function setTokens(m){
188
+ const i = STOPS.indexOf(m);
189
+ state.tokens = m;
190
+ if(i >= 0) tok.value = i;
191
+ document.getElementById('tokLabel').textContent = labelTokens(m);
192
+ updateEstimate();
193
+ }
194
+ tok.addEventListener('input', () => setTokens(STOPS[Number(tok.value)]));
195
+ document.querySelectorAll('[data-tokens]').forEach(b =>
196
+ b.addEventListener('click', () => setTokens(Number(b.dataset.tokens))));
197
+
198
+ const nameEl = document.getElementById('name');
199
+ nameEl.addEventListener('input', () => {
200
+ state.name = nameEl.value;
201
+ document.getElementById('namePreview').textContent = defaultName();
202
+ });
203
+
204
+ function defaultName(){
205
+ const s = state.name.trim().replace(/[^A-Za-z0-9._-]+/g,'-').replace(/^[-._]+|[-._]+$/g,'');
206
+ if(s) return s.slice(0,80);
207
+ const sel = document.querySelector('.pick.on[data-tier]');
208
+ return (sel ? sel.dataset.label.toLowerCase() : 'nanodex')
209
+ + '-' + labelTokens(state.tokens).toLowerCase();
210
+ }
211
+
212
+ let estTimer;
213
+ function updateEstimate(){
214
+ clearTimeout(estTimer);
215
+ estTimer = setTimeout(async () => {
216
+ try{
217
+ est = await NX.get(`/api/estimate?tier=${state.tier}&tokens=${state.tokens*1e6}`);
218
+ }catch(e){ return; }
219
+ const sel = document.querySelector('.pick.on[data-tier]');
220
+ document.getElementById('e-params').textContent = Number(est.n_params).toLocaleString();
221
+ document.getElementById('e-tier').textContent = sel ? sel.dataset.label : '';
222
+ document.getElementById('e-steps').textContent = Number(est.steps).toLocaleString();
223
+ document.getElementById('e-eta').textContent = est.eta_human;
224
+ document.getElementById('e-cal').textContent = est.calibrated
225
+ ? 'from real runs here' : 'prior — self-calibrates';
226
+ document.getElementById('e-ratio').textContent = Math.round(est.ratio) + ':1';
227
+ document.getElementById('e-chin').textContent =
228
+ est.chinchilla_x.toFixed(1) + '× Chinchilla';
229
+
230
+ const r = est.ratio;
231
+ document.getElementById('scaleNote').textContent = r < 60
232
+ ? 'Under ~60 tokens per parameter this model never uses the capacity it has. Either raise the budget or pick a smaller size.'
233
+ : (r < 150
234
+ ? 'Workable, but on the thin side for this size — 300:1 or more is where these models get interesting.'
235
+ : (r > 2000
236
+ ? 'Very deep into the over-trained regime. Still improves, but the last few hundred million tokens buy little — a larger size would use them better.'
237
+ : 'A good range for this size: well past compute-optimal, and it still finishes in reasonable time.'));
238
+ document.getElementById('namePreview').textContent = defaultName();
239
+ }, 140);
240
+ }
241
+
242
+ function fillReview(){
243
+ const sel = document.querySelector('.pick.on[data-tier]');
244
+ document.getElementById('r-name').textContent = defaultName();
245
+ document.getElementById('r-tier').textContent = sel ? sel.dataset.label : '—';
246
+ document.getElementById('r-params').textContent =
247
+ sel ? Number(sel.dataset.params).toLocaleString() : '—';
248
+ const d = sel ? sel.querySelector('.p-d').textContent.trim().split('·') : [];
249
+ document.getElementById('r-arch').textContent = 'LlamaForCausalLM';
250
+ document.getElementById('r-tok').textContent = labelTokens(state.tokens) + ' tokens';
251
+ document.getElementById('r-steps').textContent =
252
+ est ? Number(est.steps).toLocaleString() : '—';
253
+ document.getElementById('r-eta').textContent = est ? est.eta_human : '—';
254
+ }
255
+
256
+ document.getElementById('launch').addEventListener('click', async (ev) => {
257
+ const b = ev.currentTarget;
258
+ b.disabled = true; b.textContent = 'Queueing…';
259
+ try{
260
+ const r = await NX.post('/api/jobs', {
261
+ tier: state.tier, tokens: state.tokens * 1e6, name: state.name
262
+ });
263
+ location.href = '/models/' + r.id;
264
+ }catch(e){
265
+ NX.toast(NX.esc(e.message), 'err');
266
+ b.disabled = false; b.innerHTML = 'Queue this run &nbsp;→';
267
+ }
268
+ });
269
+
270
+ setTokens(state.tokens);
271
+ document.getElementById('namePreview').textContent = defaultName();
272
+ </script>
273
+ {% endblock %}
web/templates/user.html ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% extends "base.html" %}
2
+ {% set nav = 'explore' %}
3
+ {% block title %}{{ profile.display_name }} — NanoDex{% endblock %}
4
+ {% block body %}
5
+
6
+ <div class="card pad-lg" style="display:flex;gap:20px;align-items:center;flex-wrap:wrap">
7
+ {% if profile.avatar_url %}
8
+ <img src="{{ profile.avatar_url }}" alt="" style="width:72px;height:72px;
9
+ border-radius:50%;border:1px solid var(--line-2);object-fit:cover">
10
+ {% else %}
11
+ <div style="width:72px;height:72px;border-radius:50%;background:var(--panel-2);
12
+ border:1px solid var(--line-2)"></div>
13
+ {% endif %}
14
+ <div style="flex:1;min-width:200px">
15
+ <div style="display:flex;align-items:center;gap:10px;flex-wrap:wrap">
16
+ <h1 style="font-size:24px">{{ profile.display_name }}</h1>
17
+ {% if is_me %}<span class="chip done"><i></i>you</span>{% endif %}
18
+ </div>
19
+ <p class="sub" style="margin-top:3px">
20
+ <a href="https://huggingface.co/{{ profile.username }}" target="_blank"
21
+ rel="noopener" class="mono" style="color:var(--fg-2)">@{{ profile.username }}</a>
22
+ · training here since
23
+ <span id="since">{{ profile.first_seen }}</span>
24
+ </p>
25
+ </div>
26
+ <div style="display:flex;gap:8px">
27
+ {% if is_me %}<a class="btn" href="/train">+ New run</a>{% endif %}
28
+ <a class="btn ghost" href="/explore">Explore all →</a>
29
+ </div>
30
+ </div>
31
+
32
+ <div class="grid g4" style="margin-top:16px">
33
+ <div class="stat"><span>Models</span><b>{{ profile.done }}</b>
34
+ <small>{{ profile.runs }} run{{ 's' if profile.runs != 1 }} total{{ ', ' ~ profile.active ~ ' in flight' if profile.active }}</small></div>
35
+ <div class="stat"><span>Tokens trained</span>
36
+ <b>{{ "%.0fM"|format(profile.tokens / 1000000) if profile.tokens < 1000000000 else "%.2fB"|format(profile.tokens / 1000000000) }}</b>
37
+ <small>of fineweb-edu</small></div>
38
+ <div class="stat"><span>Parameters created</span>
39
+ <b>{{ "{:,}".format(profile.params) }}</b><small>across finished models</small></div>
40
+ <div class="stat"><span>Best loss</span>
41
+ <b>{{ "%.4f"|format(profile.best_loss) if profile.best_loss else "—" }}</b>
42
+ <small>{{ profile.published }} published to the Hub</small></div>
43
+ </div>
44
+
45
+ <section id="activeSec" style="margin-top:32px;display:none">
46
+ <div class="sect-h"><h2>Training now</h2></div>
47
+ <div class="runs" id="active"></div>
48
+ </section>
49
+
50
+ <section style="margin-top:32px">
51
+ <div class="sect-h">
52
+ <h2>Models</h2>
53
+ <span class="sub">open any of them to generate text</span>
54
+ </div>
55
+ <div id="models"></div>
56
+ </section>
57
+
58
+ {% endblock %}
59
+ {% block scripts %}
60
+ <script>
61
+ const USERNAME = {{ profile.username | tojson }};
62
+ const SINCE = {{ profile.first_seen | tojson }};
63
+ document.getElementById('since').textContent = SINCE
64
+ ? new Date(SINCE * 1000).toLocaleDateString(undefined,
65
+ {year:'numeric', month:'long'})
66
+ : '—';
67
+
68
+ function card(m){
69
+ return `<a class="run" href="/models/${m.id}">
70
+ <div class="run-top">
71
+ <h4>${NX.esc(m.name)}</h4>
72
+ ${NX.chip(m.status)}
73
+ </div>
74
+ <div class="run-meta">
75
+ <span class="mono">${NX.esc(m.tier_label)}</span>
76
+ <span>${Number(m.n_params).toLocaleString()} params</span>
77
+ <span><b>${NX.fmtTokens(m.tokens_seen)}</b> tokens</span>
78
+ ${m.best_loss ? `<span>loss <b>${m.best_loss.toFixed(4)}</b></span>` : ''}
79
+ ${m.pushed_repo
80
+ ? `<span class="mono" style="color:var(--accent)">${NX.esc(m.pushed_repo)}</span>`
81
+ : '<span class="muted">not published</span>'}
82
+ <span class="muted">${NX.ago(m.finished_at || m.created_at)}</span>
83
+ </div>
84
+ </a>`;
85
+ }
86
+
87
+ async function refresh(){
88
+ const d = await NX.get('/api/users/' + encodeURIComponent(USERNAME));
89
+
90
+ const sec = document.getElementById('activeSec');
91
+ sec.style.display = d.active.length ? 'block' : 'none';
92
+ document.getElementById('active').innerHTML = d.active.map(NX.runRow).join('');
93
+
94
+ document.getElementById('models').innerHTML = d.models.length
95
+ ? `<div class="runs">${d.models.map(card).join('')}</div>`
96
+ : `<div class="empty">No finished models yet.</div>`;
97
+ }
98
+ NX.poll(refresh, 6000);
99
+ </script>
100
+ {% endblock %}